You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/12/07 14:59:59 UTC
[01/50] [abbrv] ambari git commit: AMBARI-19069. Remove
'llap_queue_capacity' references from UI code. (jaimin)
Repository: ambari
Updated Branches:
refs/heads/branch-dev-patch-upgrade 06327e65e -> 90cb3a4ce
AMBARI-19069. Remove 'llap_queue_capacity' references from UI code. (jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e465c0e7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e465c0e7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e465c0e7
Branch: refs/heads/branch-dev-patch-upgrade
Commit: e465c0e7a667e8c411a1ff38480bddd49e01427c
Parents: 5901b95
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Fri Dec 2 14:39:39 2016 -0800
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Fri Dec 2 14:39:39 2016 -0800
----------------------------------------------------------------------
ambari-web/app/messages.js | 1 -
.../app/models/configs/objects/service_config_property.js | 7 -------
.../views/common/configs/widgets/slider_config_widget_view.js | 7 ++-----
3 files changed, 2 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e465c0e7/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 90ad70f..090244c 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -3200,7 +3200,6 @@ Em.I18n.translations = {
'config.infoMessage.wrong.value.for.combobox.widget': '"{0}" is not available in the list of valid values',
'config.warnMessage.outOfBoundaries.greater': 'Values greater than {0} are not recommended',
'config.warnMessage.outOfBoundaries.less': 'Values smaller than {0} are not recommended',
- 'config.warnMessage.llap_queue_capacity.max': 'LLAP will consume entire Yarn queue',
'errorMessage.config.required': 'This is required',
'errorMessage.config.number.integer': 'Must contain digits only',
http://git-wip-us.apache.org/repos/asf/ambari/blob/e465c0e7/ambari-web/app/models/configs/objects/service_config_property.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/configs/objects/service_config_property.js b/ambari-web/app/models/configs/objects/service_config_property.js
index 5d85ae0..4459946 100644
--- a/ambari-web/app/models/configs/objects/service_config_property.js
+++ b/ambari-web/app/models/configs/objects/service_config_property.js
@@ -358,13 +358,6 @@ App.ServiceConfigProperty = Em.Object.extend({
} else if ((typeof this.get('value') != 'object') && ((this.get('value') + '').length === 0)) {
var widgetType = this.get('widgetType');
this.set('errorMessage', (this.get('isRequired') && (!['test-db-connection','label'].contains(widgetType))) ? Em.I18n.t('errorMessage.config.required') : '');
- } else if (this.get('name') === 'llap_queue_capacity') {
- if (!isNaN(parseInt(this.get('value'), 10)) && parseInt(this.get('value'), 10) === 100) {
- this.set('warnMessage', Em.I18n.t('config.warnMessage.llap_queue_capacity.max'));
- } else {
- this.set('warnMessage', '');
- this.set('errorMessage', this.validateErrors(this.get('value'), this.get('name'), this.get('retypedPassword')));
- }
} else {
this.set('errorMessage', this.validateErrors(this.get('value'), this.get('name'), this.get('retypedPassword')));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e465c0e7/ambari-web/app/views/common/configs/widgets/slider_config_widget_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/configs/widgets/slider_config_widget_view.js b/ambari-web/app/views/common/configs/widgets/slider_config_widget_view.js
index 71156f4..c7e810b 100644
--- a/ambari-web/app/views/common/configs/widgets/slider_config_widget_view.js
+++ b/ambari-web/app/views/common/configs/widgets/slider_config_widget_view.js
@@ -645,11 +645,8 @@ App.SliderConfigWidgetView = App.ConfigWidgetView.extend({
return false;
}
}
- if (this.get('config.name') === 'llap_queue_capacity') {
- this.get('config').validate();
- } else {
- this.updateWarningsForCompatibilityWithWidget('');
- }
+
+ this.updateWarningsForCompatibilityWithWidget('');
return true;
}
return false;
[09/50] [abbrv] ambari git commit: AMBARI-19081. JS error on "User
RedHat Satellite" click on step1 installer (onechiporenko)
Posted by nc...@apache.org.
AMBARI-19081. JS error on "User RedHat Satellite" click on step1 installer (onechiporenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7f6ca9d0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7f6ca9d0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7f6ca9d0
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 7f6ca9d07b2f2e389d9e887853238ca3969d99ce
Parents: 8ad494e
Author: Oleg Nechiporenko <on...@apache.org>
Authored: Mon Dec 5 13:29:05 2016 +0200
Committer: Oleg Nechiporenko <on...@apache.org>
Committed: Mon Dec 5 13:29:05 2016 +0200
----------------------------------------------------------------------
ambari-web/app/styles/wizard.less | 6 ++++-
ambari-web/app/templates/wizard/step1.hbs | 2 +-
ambari-web/app/views/wizard/step1_view.js | 21 +++++++++---------
.../widgets/slider_config_widget_view_test.js | 23 --------------------
.../test/views/main/dashboard/widgets_test.js | 2 +-
5 files changed, 18 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f6ca9d0/ambari-web/app/styles/wizard.less
----------------------------------------------------------------------
diff --git a/ambari-web/app/styles/wizard.less b/ambari-web/app/styles/wizard.less
index a4dac8c..57dae45 100644
--- a/ambari-web/app/styles/wizard.less
+++ b/ambari-web/app/styles/wizard.less
@@ -554,6 +554,11 @@
}
}
}
+ .inner-table {
+ td {
+ vertical-align: middle;
+ }
+ }
.repos-panel {
.remove-icon {
color: red;
@@ -567,7 +572,6 @@
}
.repo-url input {
width: 90%;
- height: 24px;
}
#skip-validation, #use-redhat {
span.disabled {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f6ca9d0/ambari-web/app/templates/wizard/step1.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/wizard/step1.hbs b/ambari-web/app/templates/wizard/step1.hbs
index 5297d12..8179b41 100644
--- a/ambari-web/app/templates/wizard/step1.hbs
+++ b/ambari-web/app/templates/wizard/step1.hbs
@@ -144,7 +144,7 @@
{{view view.popoverView repositoryBinding="repository"}}
</td>
<td class="col-sm-8">
- <div {{bindAttr class=":repo-url repository.invalidFormatError:textfield-error repository.invalidError:textfield-error"}}>
+ <div {{bindAttr class=":repo-url repository.invalidFormatError:has-error repository.invalidError:has-error"}}>
{{view Ember.TextField placeholderBinding="repository.placeholder" valueBinding="repository.baseUrl" disabledBinding="controller.selectedStack.useRedhatSatellite" classNames="form-control"}}
{{#if controller.selectedStack.usePublicRepo}}
<i {{bindAttr class="repository.undo::invisible :icon-undo"}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f6ca9d0/ambari-web/app/views/wizard/step1_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/wizard/step1_view.js b/ambari-web/app/views/wizard/step1_view.js
index d5cbbb5..161d794 100644
--- a/ambari-web/app/views/wizard/step1_view.js
+++ b/ambari-web/app/views/wizard/step1_view.js
@@ -226,17 +226,18 @@ App.WizardStep1View = Em.View.extend({
checkedBinding: 'controller.selectedStack.useRedhatSatellite',
disabledBinding: 'controller.selectedStack.usePublicRepo',
click: function () {
- // click triggered before value is toggled, so if-statement is inverted
- if (this.get('disabled')) return;
- if (!this.get('controller.selectedStack.useRedhatSatellite')) {
- App.ModalPopup.show({
- header: Em.I18n.t('common.important'),
- secondary: false,
- bodyClass: Ember.View.extend({
- template: Ember.Handlebars.compile(Em.I18n.t('installer.step1.advancedRepo.useRedhatSatellite.warning'))
- })
- });
+ if (!this.get('disabled')) {
+ this.toggleProperty('controller.selectedStack.useRedhatSatellite');
+ if (this.get('controller.selectedStack.useRedhatSatellite')) {
+ App.ModalPopup.show({
+ header: Em.I18n.t('common.important'),
+ encodeBody: false,
+ secondary: false,
+ body: Em.I18n.t('installer.step1.advancedRepo.useRedhatSatellite.warning')
+ });
+ }
}
+ return false;
}
}),
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f6ca9d0/ambari-web/test/views/common/configs/widgets/slider_config_widget_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/common/configs/widgets/slider_config_widget_view_test.js b/ambari-web/test/views/common/configs/widgets/slider_config_widget_view_test.js
index a6b25ce..5cc055c 100644
--- a/ambari-web/test/views/common/configs/widgets/slider_config_widget_view_test.js
+++ b/ambari-web/test/views/common/configs/widgets/slider_config_widget_view_test.js
@@ -636,29 +636,6 @@ describe('App.SliderConfigWidgetView', function () {
expect(viewInt.get('issueMessage')).to.equal('');
});
- describe('llap_queue_capacity property', function() {
- beforeEach(function() {
- viewInt.set('config.name', 'llap_queue_capacity');
- });
- it('should validate and warn about llap issue when value is 100%', function() {
- viewInt.set('config.stackConfigProperty.valueAttributes.maximum', 100);
- viewInt.set('config.value', '100');
- viewInt.set('config.errorMessage', '');
- viewInt.set('config.warnMessage', '');
- viewInt.set('config.widgetType', 'slider');
- assert.isTrue(viewInt.isValueCompatibleWithWidget(), 'value should be compatible with widget');
- assert.equal(viewInt.get('config.warnMessage'), Em.I18n.t('config.warnMessage.llap_queue_capacity.max'), 'warn message validation');
- });
-
- it('should pass validation because llap < 100', function() {
- viewInt.set('config.stackConfigProperty.valueAttributes.maximum', 100);
- viewInt.set('config.value', '99');
- viewInt.set('config.errorMessage', '');
- viewInt.set('config.warnMessage', '');
- assert.isTrue(viewInt.isValueCompatibleWithWidget(), 'value should be compatible with widget');
- assert.equal(viewInt.get('config.warnMessage'), '', 'warn message validation');
- });
- });
});
describe('#formatTickLabel', function () {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f6ca9d0/ambari-web/test/views/main/dashboard/widgets_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/main/dashboard/widgets_test.js b/ambari-web/test/views/main/dashboard/widgets_test.js
index 707087a..b2d154b 100644
--- a/ambari-web/test/views/main/dashboard/widgets_test.js
+++ b/ambari-web/test/views/main/dashboard/widgets_test.js
@@ -223,7 +223,7 @@ describe('App.MainDashboardWidgetsView', function () {
expect(view.get('visibleWidgets')).to.be.eql([Em.Object.create({
id: 1,
threshold: [],
- viewClass: App['NameNodeHeapPieChartView'],
+ viewClass: App.NameNodeHeapPieChartView,
sourceName: 'HDFS',
title: Em.I18n.t('dashboard.widgets.NameNodeHeap')
})]);
[12/50] [abbrv] ambari git commit: AMBARI-19083. After ambari only
upgrade Enable security(Configure Ambari Identity) was failed because could
not run "ambari-sudo.sh" (aonishuk)
Posted by nc...@apache.org.
AMBARI-19083. After ambari only upgrade Enable security(Configure Ambari Identity) was failed because could not run "ambari-sudo.sh" (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/89b9cf5b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/89b9cf5b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/89b9cf5b
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 89b9cf5beb1304b0d90dad00b371947a3c467e8a
Parents: 08342b3
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Dec 5 15:23:58 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Dec 5 15:23:58 2016 +0200
----------------------------------------------------------------------
ambari-server/src/main/python/ambari_server_main.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/89b9cf5b/ambari-server/src/main/python/ambari_server_main.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py
index b642cea..dbcafa4 100644
--- a/ambari-server/src/main/python/ambari_server_main.py
+++ b/ambari-server/src/main/python/ambari_server_main.py
@@ -28,6 +28,7 @@ from ambari_commons.logging_utils import get_debug_mode, print_warning_msg, prin
from ambari_commons.os_check import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons.os_utils import is_root, run_os_command
+from ambari_server.ambariPath import AmbariPath
from ambari_server.dbConfiguration import ensure_dbms_is_running, ensure_jdbc_driver_is_installed
from ambari_server.serverConfiguration import configDefaults, find_jdk, get_ambari_properties, \
get_conf_dir, get_is_persisted, get_is_secure, get_java_exe_path, get_original_master_key, read_ambari_user, \
@@ -121,6 +122,8 @@ AMBARI_SERVER_NOT_STARTED_MSG = "Ambari Server java process hasn't been started
ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
ULIMIT_OPEN_FILES_DEFAULT = 10000
+AMBARI_ENV_FILE = AmbariPath.get("/var/lib/ambari-server/ambari-env.sh")
+
@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
def ensure_server_security_is_configured():
pass
@@ -179,9 +182,10 @@ def generate_child_process_param_list(ambari_user, java_exe, class_path,
# from subprocess, we have to skip --login option of su command. That's why
# we change dir to / (otherwise subprocess can face with 'permission denied'
# errors while trying to list current directory
- cmd = "{ulimit_cmd} ; {su} {ambari_user} -s {sh_shell} -c '{command}'".format(ulimit_cmd=ulimit_cmd,
+ cmd = "{ulimit_cmd} ; {su} {ambari_user} -s {sh_shell} -c 'source {ambari_env_file} ; {command}'".format(ulimit_cmd=ulimit_cmd,
su=locate_file('su', '/bin'), ambari_user=ambari_user,
- sh_shell=locate_file('sh', '/bin'), command=command)
+ sh_shell=locate_file('sh', '/bin'), command=command,
+ ambari_env_file=AMBARI_ENV_FILE)
else:
cmd = "{ulimit_cmd} ; {command}".format(ulimit_cmd=ulimit_cmd, command=command)
[45/50] [abbrv] ambari git commit: AMBARI-18966. Add check to ensure
we do not have @Transactional annotations on private methods. (Attila
Doroszlai via stoader)
Posted by nc...@apache.org.
AMBARI-18966. Add check to ensure we do not have @Transactional annotations on private methods. (Attila Doroszlai via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/304a5bf6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/304a5bf6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/304a5bf6
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 304a5bf6027efa9e70b1deaba325d54e586e4292
Parents: 6806edc
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Wed Dec 7 10:45:54 2016 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Wed Dec 7 10:45:54 2016 +0100
----------------------------------------------------------------------
ambari-project/pom.xml | 41 +++++++++++++++
ambari-server/checkstyle.xml | 17 ++++++
ambari-server/pom.xml | 4 ++
.../internal/AlertTargetResourceProvider.java | 2 +-
.../ClusterStackVersionResourceProvider.java | 4 +-
.../internal/UpgradeResourceProvider.java | 2 +-
.../apache/ambari/server/orm/dao/AlertsDAO.java | 6 +--
.../apache/ambari/server/state/ServiceImpl.java | 2 +-
.../server/state/cluster/ClustersImpl.java | 2 +-
.../ambari/server/state/host/HostImpl.java | 2 +-
.../services/RetryUpgradeActionService.java | 2 +-
.../svccomphost/ServiceComponentHostImpl.java | 2 +-
.../server/upgrade/UpgradeCatalog240.java | 4 +-
pom.xml | 1 +
utility/pom.xml | 33 +++++++++++-
...AvoidTransactionalOnPrivateMethodsCheck.java | 55 ++++++++++++++++++++
.../src/main/resources/checkstyle_packages.xml | 15 ++++++
...dTransactionalOnPrivateMethodsCheckTest.java | 49 +++++++++++++++++
.../InputTransactionalOnPrivateMethods.java | 46 ++++++++++++++++
19 files changed, 274 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-project/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml
index 7f273eb..b00be38 100644
--- a/ambari-project/pom.xml
+++ b/ambari-project/pom.xml
@@ -30,6 +30,8 @@
<ambari.dir>${project.parent.basedir}</ambari.dir>
<powermock.version>1.6.3</powermock.version>
<jetty.version>8.1.19.v20160209</jetty.version>
+ <checkstyle.version>6.19</checkstyle.version> <!-- last version that does not require Java 8 -->
+ <checkstyle.skip>false</checkstyle.skip>
</properties>
<profiles>
<profile>
@@ -477,6 +479,11 @@
<artifactId>jline</artifactId>
<version>2.11</version>
</dependency>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<build>
@@ -486,6 +493,40 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>2.17</version>
+ <executions>
+ <execution>
+ <id>checkstyle</id>
+ <phase>test</phase>
+ <configuration>
+ <configLocation>${project.basedir}/checkstyle.xml</configLocation>
+ <encoding>UTF-8</encoding>
+ <consoleOutput>true</consoleOutput>
+ <failsOnError>true</failsOnError>
+ <linkXRef>false</linkXRef>
+ <skip>${checkstyle.skip}</skip>
+ </configuration>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>utility</groupId>
+ <artifactId>utility</artifactId>
+ <version>1.0.0.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/checkstyle.xml
----------------------------------------------------------------------
diff --git a/ambari-server/checkstyle.xml b/ambari-server/checkstyle.xml
new file mode 100644
index 0000000..81f6380
--- /dev/null
+++ b/ambari-server/checkstyle.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<!-- Licensed under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may obtain
+ a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+ required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ See accompanying LICENSE file. -->
+<!DOCTYPE module PUBLIC
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+ <module name="TreeWalker">
+ <module name="AvoidTransactionalOnPrivateMethodsCheck"/>
+ </module>
+</module>
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 69ab9d0..e8de84e 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -692,6 +692,10 @@
<artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
</plugins>
<resources>
<resource>
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
index 8e5674e..4010528 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
@@ -374,7 +374,7 @@ public class AlertTargetResourceProvider extends
*/
@Transactional
@SuppressWarnings("unchecked")
- private void updateAlertTargets(long alertTargetId,
+ void updateAlertTargets(long alertTargetId,
Map<String, Object> requestMap)
throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index 42aae84..d417ec2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -442,7 +442,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- private void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
+ void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
String desiredRepoVersion, RepositoryVersionState repoState)
throws AmbariException, SystemException {
final String clusterName = cluster.getClusterName();
@@ -482,7 +482,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- private RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
+ RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
List<Host> hosts, RepositoryVersionEntity repoVersionEnt, Map<String, Object> propertyMap)
throws AmbariException, SystemException {
final AmbariManagementController managementController = getManagementController();
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 5931743..67be152 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -972,7 +972,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
@Transactional
- private UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
+ UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
RequestStageContainer request,
UpgradeEntity upgradeEntity) throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
index d187947..e8c4b5f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
@@ -1492,7 +1492,7 @@ public class AlertsDAO implements Cleanable {
* @return a long representing the number of affected (deleted) records
*/
@Transactional
- private int cleanAlertNoticesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
+ int cleanAlertNoticesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
LOG.info("Deleting AlertNotice entities before date " + new Date(beforeDateMillis));
EntityManager entityManager = m_entityManagerProvider.get();
List<Integer> ids = findAllAlertHistoryIdsBeforeDate(clusterId, beforeDateMillis);
@@ -1523,7 +1523,7 @@ public class AlertsDAO implements Cleanable {
* @return a long representing the number of affected (deleted) records
*/
@Transactional
- private int cleanAlertCurrentsForClusterBeforeDate(long clusterId, long beforeDateMillis) {
+ int cleanAlertCurrentsForClusterBeforeDate(long clusterId, long beforeDateMillis) {
LOG.info("Deleting AlertCurrent entities before date " + new Date(beforeDateMillis));
EntityManager entityManager = m_entityManagerProvider.get();
List<Integer> ids = findAllAlertHistoryIdsBeforeDate(clusterId, beforeDateMillis);
@@ -1553,7 +1553,7 @@ public class AlertsDAO implements Cleanable {
*/
@Transactional
- private int cleanAlertHistoriesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
+ int cleanAlertHistoriesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
return executeQuery("AlertHistoryEntity.removeInClusterBeforeDate", AlertHistoryEntity.class, clusterId, beforeDateMillis);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
index f87b99c..e223eed 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
@@ -451,7 +451,7 @@ public class ServiceImpl implements Service {
}
@Transactional
- private void persistEntities(ClusterServiceEntity serviceEntity) {
+ void persistEntities(ClusterServiceEntity serviceEntity) {
long clusterId = cluster.getClusterId();
ClusterEntity clusterEntity = clusterDAO.findById(clusterId);
serviceEntity.setClusterEntity(clusterEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 8e5f37e..1de10f9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@ -183,7 +183,7 @@ public class ClustersImpl implements Clusters {
*/
@Inject
@Transactional
- private void loadClustersAndHosts() {
+ void loadClustersAndHosts() {
List<HostEntity> hostEntities = hostDAO.findAll();
for (HostEntity hostEntity : hostEntities) {
Host host = hostFactory.create(hostEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index dd5e635..a444f5d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@ -954,7 +954,7 @@ public class HostImpl implements Host {
}
@Transactional
- private void persistEntities(HostEntity hostEntity) {
+ void persistEntities(HostEntity hostEntity) {
hostDAO.create(hostEntity);
if (!hostEntity.getClusterEntities().isEmpty()) {
for (ClusterEntity clusterEntity : hostEntity.getClusterEntities()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
index 1ea5558..a92aa04 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
@@ -190,7 +190,7 @@ public class RetryUpgradeActionService extends AbstractScheduledService {
* @param requestId Request Id to search tasks for.
*/
@Transactional
- private void retryHoldingCommandsInRequest(Long requestId) {
+ void retryHoldingCommandsInRequest(Long requestId) {
if (requestId == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index c1655aa..5225598 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -1313,7 +1313,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
}
@Transactional
- private void persistEntities(HostEntity hostEntity, HostComponentStateEntity stateEntity,
+ void persistEntities(HostEntity hostEntity, HostComponentStateEntity stateEntity,
HostComponentDesiredStateEntity desiredStateEntity) {
ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName(
serviceComponent.getClusterId(), serviceComponent.getServiceName(),
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index c113fff..b7e248a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -1563,7 +1563,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
* @throws SQLException
*/
@Transactional
- private void updateServiceComponentDesiredStateTableDDL() throws SQLException {
+ void updateServiceComponentDesiredStateTableDDL() throws SQLException {
if (dbAccessor.tableHasPrimaryKey(SERVICE_COMPONENT_DS_TABLE, ID)) {
LOG.info("Skipping {} table Primary Key modifications since the new {} column already exists",
SERVICE_COMPONENT_DS_TABLE, ID);
@@ -2785,7 +2785,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
* instead of cluster_name
*/
@Transactional
- private void updateViewInstanceTable() throws SQLException {
+ void updateViewInstanceTable() throws SQLException {
try {
if (Long.class.equals(dbAccessor.getColumnClass(VIEWINSTANCE_TABLE, CLUSTER_HANDLE_COLUMN))) {
LOG.info(String.format("%s column is already numeric. Skipping an update of %s table.", CLUSTER_HANDLE_COLUMN, VIEWINSTANCE_TABLE));
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bd0aa7b..aeb1467 100644
--- a/pom.xml
+++ b/pom.xml
@@ -266,6 +266,7 @@
<exclude>contrib/agent-simulator/docker_image/package_list.txt</exclude>
<exclude>contrib/agent-simulator/config/cluster.txt</exclude>
<exclude>version</exclude>
+ <exclude>**/target/surefire-reports/</exclude>
<!--IDE and GIT files-->
<exclude>**/.idea/</exclude>
<exclude>**/.classpath/</exclude>
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/utility/pom.xml
----------------------------------------------------------------------
diff --git a/utility/pom.xml b/utility/pom.xml
index 2febb83..6f60206 100644
--- a/utility/pom.xml
+++ b/utility/pom.xml
@@ -20,6 +20,13 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-project</artifactId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ <relativePath>../ambari-project</relativePath>
+ </parent>
+
<artifactId>utility</artifactId>
<groupId>utility</groupId>
<version>1.0.0.0-SNAPSHOT</version>
@@ -28,9 +35,25 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.12</version>
<scope>compile</scope> <!-- has to be compile-time dependency on junit -->
</dependency>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <type>test-jar</type>
+ <version>${checkstyle.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>19.0</version> <!-- required for checkstyle -->
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -45,6 +68,14 @@
</configuration>
</plugin>
<plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>${project.parent.basedir}/src/main/assemblies/empty.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>2.0.1</version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/utility/src/main/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheck.java
----------------------------------------------------------------------
diff --git a/utility/src/main/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheck.java b/utility/src/main/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheck.java
new file mode 100644
index 0000000..80da103
--- /dev/null
+++ b/utility/src/main/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheck.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.checkstyle;
+
+import com.puppycrawl.tools.checkstyle.api.AbstractCheck;
+import com.puppycrawl.tools.checkstyle.api.DetailAST;
+import com.puppycrawl.tools.checkstyle.api.TokenTypes;
+
+/**
+ * Detects private methods annotated as <code>Transactional</code>.
+ * See https://github.com/google/guice/wiki/Transactions for why this should be
+ * avoided.
+ */
+public class AvoidTransactionalOnPrivateMethodsCheck extends AbstractCheck {
+
+ private static final String ANNOTATION_NAME = "Transactional";
+ public static final String MSG_TRANSACTIONAL_ON_PRIVATE_METHOD = "@" + ANNOTATION_NAME + " should not be used on private methods";
+
+ @Override
+ public int[] getDefaultTokens() {
+ return new int[] { TokenTypes.METHOD_DEF };
+ }
+
+ @Override
+ public void visitToken(DetailAST ast) {
+ DetailAST modifiers = ast.findFirstToken(TokenTypes.MODIFIERS);
+ if (modifiers.findFirstToken(TokenTypes.LITERAL_PRIVATE) != null) {
+ DetailAST annotation = modifiers.findFirstToken(TokenTypes.ANNOTATION);
+ while (annotation != null) {
+ DetailAST name = annotation.findFirstToken(TokenTypes.IDENT);
+ if (name != null && ANNOTATION_NAME.equals(name.getText())) {
+ log(ast.getLineNo(), MSG_TRANSACTIONAL_ON_PRIVATE_METHOD);
+ break;
+ }
+ annotation = annotation.getNextSibling();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/utility/src/main/resources/checkstyle_packages.xml
----------------------------------------------------------------------
diff --git a/utility/src/main/resources/checkstyle_packages.xml b/utility/src/main/resources/checkstyle_packages.xml
new file mode 100644
index 0000000..8cf4a44
--- /dev/null
+++ b/utility/src/main/resources/checkstyle_packages.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may obtain
+ a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+ required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ See accompanying LICENSE file. -->
+<!DOCTYPE checkstyle-packages PUBLIC
+ "-//Puppy Crawl//DTD Package Names 1.0//EN"
+ "http://www.puppycrawl.com/dtds/packages_1_0.dtd">
+<checkstyle-packages>
+ <package name="org.apache.ambari.checkstyle"/>
+</checkstyle-packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/utility/src/test/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheckTest.java
----------------------------------------------------------------------
diff --git a/utility/src/test/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheckTest.java b/utility/src/test/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheckTest.java
new file mode 100644
index 0000000..14d7486
--- /dev/null
+++ b/utility/src/test/java/org/apache/ambari/checkstyle/AvoidTransactionalOnPrivateMethodsCheckTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.checkstyle;
+
+import static org.apache.ambari.checkstyle.AvoidTransactionalOnPrivateMethodsCheck.MSG_TRANSACTIONAL_ON_PRIVATE_METHOD;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.puppycrawl.tools.checkstyle.BaseCheckTestSupport;
+import com.puppycrawl.tools.checkstyle.DefaultConfiguration;
+
+import org.junit.Test;
+
+public class AvoidTransactionalOnPrivateMethodsCheckTest extends BaseCheckTestSupport {
+
+ @Override
+ protected String getPath(String filename) throws IOException {
+ return new File("src/test/resources/org/apache/ambari/checkstyle/" + filename)
+ .getCanonicalPath();
+ }
+
+ @Test
+ public void transactionalOnPrivateMethod() throws Exception {
+ final DefaultConfiguration config = createCheckConfig(AvoidTransactionalOnPrivateMethodsCheck.class);
+ final String[] expected = {
+ "32: " + MSG_TRANSACTIONAL_ON_PRIVATE_METHOD,
+ "41: " + MSG_TRANSACTIONAL_ON_PRIVATE_METHOD,
+ };
+
+ verify(config, getPath("InputTransactionalOnPrivateMethods.java"), expected);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/304a5bf6/utility/src/test/resources/org/apache/ambari/checkstyle/InputTransactionalOnPrivateMethods.java
----------------------------------------------------------------------
diff --git a/utility/src/test/resources/org/apache/ambari/checkstyle/InputTransactionalOnPrivateMethods.java b/utility/src/test/resources/org/apache/ambari/checkstyle/InputTransactionalOnPrivateMethods.java
new file mode 100644
index 0000000..2ea5444
--- /dev/null
+++ b/utility/src/test/resources/org/apache/ambari/checkstyle/InputTransactionalOnPrivateMethods.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.checkstyle;
+
+import com.google.inject.persist.Transactional;
+
+/**
+ * Input file for AvoidTransactionalOnPrivateMethodsCheckTest.
+ */
+public class InputTransactionalOnPrivateMethods {
+
+ @Transactional
+ public void publicMethodWithTransactional() {
+ ;
+ }
+
+ @Transactional
+ private void privateMethodWithTransactional() {
+ ;
+ }
+
+ private void privateMethodWithoutTransactional() {
+ ;
+ }
+
+ @Transactional
+ private void otherPrivateMethodWithTransactional() {
+ ;
+ }
+
+}
[38/50] [abbrv] ambari git commit: AMBARI-19093. HDP 3.0 support for
HDFS with configs, kerberos, widgets, metrics, quicklinks,
and themes (alejandro)
Posted by nc...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml
new file mode 100644
index 0000000..30c49c7
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/metainfo.xml
@@ -0,0 +1,190 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>HDFS</name>
+ <displayName>HDFS</displayName>
+ <comment>Apache Hadoop Distributed File System</comment>
+ <version>2.7.1.3.0</version>
+ <extends>common-services/HDFS/2.1.0.2.0</extends>
+
+ <components>
+ <!-- NFS Gateway was added in HDP 2.3. -->
+ <component>
+ <name>NFS_GATEWAY</name>
+ <displayName>NFSGateway</displayName>
+ <cardinality>0+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/nfsgateway.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+
+ <component>
+ <name>ZKFC</name>
+ <versionAdvertised>true</versionAdvertised>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>rpcbind</name>
+ <condition>should_install_rpcbind</condition>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7,amazon2015,redhat6,suse11</osFamily>
+ <packages>
+ <package>
+ <name>hadoop_${stack_version}</name>
+ </package>
+ <package>
+ <name>hadoop_${stack_version}-client</name>
+ </package>
+ <package>
+ <name>snappy</name>
+ </package>
+ <package>
+ <name>snappy-devel</name>
+ </package>
+ <package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadoop_${stack_version}-libhdfs</name>
+ </package>
+ </packages>
+ </osSpecific>
+
+ <osSpecific>
+ <osFamily>suse12</osFamily>
+ <packages>
+ <package>
+ <name>hadoop_${stack_version}</name>
+ </package>
+ <package>
+ <name>hadoop_${stack_version}-client</name>
+ </package>
+ <package>
+ <name>snappy</name>
+ </package>
+ <package>
+ <name>snappy-devel</name>
+ </package>
+ <package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadoop_${stack_version}-libhdfs</name>
+ </package>
+ </packages>
+ </osSpecific>
+
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>hadoop-${stack_version}-client</name>
+ </package>
+ <package>
+ <name>hadoop-${stack_version}-hdfs-datanode</name>
+ </package>
+ <package>
+ <name>hadoop-${stack_version}-hdfs-journalnode</name>
+ </package>
+ <package>
+ <name>hadoop-${stack_version}-hdfs-namenode</name>
+ </package>
+ <package>
+ <name>hadoop-${stack_version}-hdfs-secondarynamenode</name>
+ </package>
+ <package>
+ <name>hadoop-${stack_version}-hdfs-zkfc</name>
+ </package>
+ <package>
+ <name>libsnappy1</name>
+ </package>
+ <package>
+ <name>libsnappy-dev</name>
+ </package>
+ <package>
+ <name>hadooplzo-${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>libhdfs0-${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/quicklinks/quicklinks.json
new file mode 100644
index 0000000..5318ba0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/quicklinks/quicklinks.json
@@ -0,0 +1,80 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"https",
+ "checks":[
+ {
+ "property":"dfs.http.policy",
+ "desired":"HTTPS_ONLY",
+ "site":"hdfs-site"
+ }
+ ]
+ },
+
+ "links": [
+ {
+ "name": "namenode_ui",
+ "label": "NameNode UI",
+ "component_name": "NAMENODE",
+ "url":"%@://%@:%@",
+ "requires_user_name": "false",
+ "port":{
+ "http_property": "dfs.namenode.http-address",
+ "http_default_port": "50070",
+ "https_property": "dfs.namenode.https-address",
+ "https_default_port": "50470",
+ "regex": "\\w*:(\\d+)",
+ "site": "hdfs-site"
+ }
+ },
+ {
+ "name": "namenode_logs",
+ "label": "NameNode Logs",
+ "component_name": "NAMENODE",
+ "url":"%@://%@:%@/logs",
+ "requires_user_name": "false",
+ "port":{
+ "http_property": "dfs.namenode.http-address",
+ "http_default_port": "50070",
+ "https_property": "dfs.namenode.https-address",
+ "https_default_port": "50470",
+ "regex": "\\w*:(\\d+)",
+ "site": "hdfs-site"
+ }
+ },
+ {
+ "name": "namenode_jmx",
+ "label": "NameNode JMX",
+ "component_name": "NAMENODE",
+ "url":"%@://%@:%@/jmx",
+ "requires_user_name": "false",
+ "port":{
+ "http_property": "dfs.namenode.http-address",
+ "http_default_port": "50070",
+ "https_property": "dfs.namenode.https-address",
+ "https_default_port": "50470",
+ "regex": "\\w*:(\\d+)",
+ "site": "hdfs-site"
+ }
+ },
+ {
+ "name": "Thread Stacks",
+ "label": "Thread Stacks",
+ "component_name": "NAMENODE",
+ "url":"%@://%@:%@/stacks",
+ "requires_user_name": "false",
+ "port":{
+ "http_property": "dfs.namenode.http-address",
+ "http_default_port": "50070",
+ "https_property": "dfs.namenode.https-address",
+ "https_default_port": "50470",
+ "regex": "\\w*:(\\d+)",
+ "site": "hdfs-site"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/themes/theme.json
new file mode 100644
index 0000000..6f2b797
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/themes/theme.json
@@ -0,0 +1,179 @@
+{
+ "name": "default",
+ "description": "Default theme for HDFS service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "2",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-namenode",
+ "display-name": "NameNode",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-namenode-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "section-datanode",
+ "display-name": "DataNode",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-datanode-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "hdfs-site/dfs.namenode.name.dir",
+ "subsection-name": "subsection-namenode-col1"
+ },
+ {
+ "config": "hadoop-env/namenode_heapsize",
+ "subsection-name": "subsection-namenode-col1"
+ },
+ {
+ "config": "hdfs-site/dfs.namenode.handler.count",
+ "subsection-name": "subsection-namenode-col1"
+ },
+ {
+ "config": "hdfs-site/dfs.namenode.safemode.threshold-pct",
+ "subsection-name": "subsection-namenode-col1"
+ },
+ {
+ "config": "hdfs-site/dfs.datanode.data.dir",
+ "subsection-name": "subsection-datanode-col1"
+ },
+ {
+ "config": "hdfs-site/dfs.datanode.failed.volumes.tolerated",
+ "subsection-name": "subsection-datanode-col1"
+ },
+ {
+ "config": "hadoop-env/dtnode_heapsize",
+ "subsection-name": "subsection-datanode-col1"
+ },
+ {
+ "config": "hdfs-site/dfs.datanode.max.transfer.threads",
+ "subsection-name": "subsection-datanode-col1"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "hdfs-site/dfs.namenode.name.dir",
+ "widget": {
+ "type": "directories"
+ }
+ },
+ {
+ "config": "hdfs-site/dfs.namenode.safemode.threshold-pct",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "percent"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hdfs-site/dfs.namenode.handler.count",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hadoop-env/namenode_heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hdfs-site/dfs.datanode.failed.volumes.tolerated",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hdfs-site/dfs.datanode.data.dir",
+ "widget": {
+ "type": "directories"
+ }
+ },
+ {
+ "config": "hadoop-env/dtnode_heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hdfs-site/dfs.datanode.max.transfer.threads",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
+
[20/50] [abbrv] ambari git commit: AMBARI-19039: In stack version,
build number should not be mandatory (Amruta Borkar via dili)
Posted by nc...@apache.org.
AMBARI-19039: In stack version, build number should not be mandatory (Amruta Borkar via dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b5122e25
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b5122e25
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b5122e25
Branch: refs/heads/branch-dev-patch-upgrade
Commit: b5122e258bfe5a2e56511eb9dd29a70cf1260a9c
Parents: 50ead4d
Author: Di Li <di...@apache.org>
Authored: Mon Dec 5 16:03:50 2016 -0500
Committer: Di Li <di...@apache.org>
Committed: Mon Dec 5 16:03:50 2016 -0500
----------------------------------------------------------------------
.../python/resource_management/libraries/functions/copy_tarball.py | 2 +-
.../libraries/functions/dynamic_variable_interpretation.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5122e25/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 519c88b..31a9be4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -188,7 +188,7 @@ def _get_single_version_from_stack_select():
Logger.error("Could not verify stack version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_stack_versions_cmd, str(code), str(out)))
return None
- matches = re.findall(r"([\d\.]+\-\d+)", out)
+ matches = re.findall(r"([\d\.]+(?:-\d+)?)", out)
if matches and len(matches) == 1:
stack_version = matches[0]
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5122e25/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py b/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
index ca8fe19..74db04a 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
@@ -185,7 +185,7 @@ def copy_tarballs_to_hdfs(tarball_prefix, stack_select_component_name, component
(get_stack_version_cmd, str(code), str(out)))
return 1
- matches = re.findall(r"([\d\.]+\-\d+)", out)
+ matches = re.findall(r"([\d\.]+(?:-\d+)?)", out)
stack_version = matches[0] if matches and len(matches) > 0 else None
if not stack_version:
[29/50] [abbrv] ambari git commit: AMBARI-19110. "Cluster Load"
widget in ambari-dashboard, download as CSV is not working (alexantonenko)
Posted by nc...@apache.org.
AMBARI-19110. "Cluster Load" widget in ambari-dashboard, download as CSV is not working (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f297510
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f297510
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f297510
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 0f297510640f24db5c11bfbabaa390bb29e619a6
Parents: 2cd12ce
Author: Alex Antonenko <hi...@gmail.com>
Authored: Tue Dec 6 18:29:00 2016 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Tue Dec 6 19:04:30 2016 +0200
----------------------------------------------------------------------
.../common/widgets/export_metrics_mixin.js | 28 ++++++++++++++++++--
1 file changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f297510/ambari-web/app/mixins/common/widgets/export_metrics_mixin.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/widgets/export_metrics_mixin.js b/ambari-web/app/mixins/common/widgets/export_metrics_mixin.js
index 7517efd..aa4f77c 100644
--- a/ambari-web/app/mixins/common/widgets/export_metrics_mixin.js
+++ b/ambari-web/app/mixins/common/widgets/export_metrics_mixin.js
@@ -85,6 +85,7 @@ App.ExportMetricsMixin = Em.Mixin.create({
ticksNumber,
metricsNumber,
metricsArray;
+ this.checkGraphDataForValidity(data);
titles = data.map(function (item) {
return displayUnit ? item.name + ' (' + displayUnit + ')' : item.name;
}, this);
@@ -95,12 +96,35 @@ App.ExportMetricsMixin = Em.Mixin.create({
for (var i = 0; i < ticksNumber; i++) {
metricsArray.push([data[0].data[i][1]]);
for (var j = 0; j < metricsNumber; j++) {
- metricsArray[i + 1].push(data[j].data[i][0]);
+ metricsArray[i + 1].push(data[j].data[i][0]);
};
- }
+ };
return stringUtils.arrayToCSV(metricsArray);
},
+ checkGraphDataForValidity: function (data) {
+ data.sort(function (a, b) {
+ return b.data.length - a.data.length
+ });
+
+ var maxLength = data[0].data.length;
+
+ for (var i = 1; i < data.length; i ++) {
+ if (data[i].data.length !== maxLength) this.fillGraphDataArrayWithMockedData(data[i], maxLength);
+ }
+ },
+
+ fillGraphDataArrayWithMockedData: function (dataArray, neededLength) {
+ var startIndex = dataArray.data.length,
+ timestampInterval = dataArray.data[2][1] - dataArray.data[1][1];
+
+ for (var i = startIndex; i < neededLength; i++) {
+ var previousTimestamp = dataArray.data[i - 1][1];
+
+ dataArray.data.push([null, previousTimestamp + timestampInterval]);
+ }
+ },
+
jsonReplacer: function () {
var displayUnit = this.get('targetView.displayUnit');
return function (key, value) {
[23/50] [abbrv] ambari git commit: AMBARI-19088 : Make sure AMS in
embedded mode cannot have more than 1 deployed instance. (avijayan)
Posted by nc...@apache.org.
AMBARI-19088 : Make sure AMS in embedded mode cannot have more than 1 deployed instance. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c711850a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c711850a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c711850a
Branch: refs/heads/branch-dev-patch-upgrade
Commit: c711850a8e53d3c838a4f7c5b9b81f7b133afbdf
Parents: edf8ab2
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Dec 5 14:30:16 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Mon Dec 5 14:30:16 2016 -0800
----------------------------------------------------------------------
.../0.1.0/package/scripts/metrics_collector.py | 2 ++
.../AMBARI_METRICS/0.1.0/package/scripts/params.py | 9 +++++++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c711850a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
index c26eafc..7073de6 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
@@ -38,6 +38,8 @@ class AmsCollector(Script):
def configure(self, env, action = None):
import params
env.set_params(params)
+ if action == 'start' and params.embedded_mode_multiple_instances:
+ raise Fail("AMS in embedded mode cannot have more than 1 instance. Delete all but 1 instances or switch to Distributed mode ")
hbase('master', action)
hbase('regionserver', action)
ams(name='collector')
http://git-wip-us.apache.org/repos/asf/ambari/blob/c711850a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 4bda033..529cf4c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -55,10 +55,16 @@ pass
#AMBARI_METRICS data
ams_pid_dir = status_params.ams_collector_pid_dir
-
+is_ams_distributed = config['configurations']['ams-site']['timeline.metrics.service.operation.mode'] == 'distributed'
ams_collector_script = "/usr/sbin/ambari-metrics-collector"
ams_collector_pid_dir = status_params.ams_collector_pid_dir
ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+ams_collector_list = default("/clusterHostInfo/metrics_collector_hosts", [])
+embedded_mode_multiple_instances = False
+
+if not is_ams_distributed and len(ams_collector_list) > 1:
+ embedded_mode_multiple_instances = True
+
failover_strategy_blacklisted_interval_seconds = default("/configurations/ams-env/failover_strategy_blacklisted_interval", "600")
failover_strategy = default("/configurations/ams-site/failover.strategy", "round-robin")
if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
@@ -180,7 +186,6 @@ hbase_pid_dir = status_params.hbase_pid_dir
is_hbase_distributed = config['configurations']['ams-hbase-site']['hbase.cluster.distributed']
is_local_fs_rootdir = hbase_root_dir.startswith('file://')
-is_ams_distributed = config['configurations']['ams-site']['timeline.metrics.service.operation.mode'] == 'distributed'
# security is disabled for embedded mode, when HBase is backed by file
security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']
[05/50] [abbrv] ambari git commit: AMBARI-19073 - Ambari Contrib View
Has Inconsistent Version in pom.xml (jonathanhurley)
Posted by nc...@apache.org.
AMBARI-19073 - Ambari Contrib View Has Inconsistent Version in pom.xml (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3096c79d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3096c79d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3096c79d
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 3096c79dfa666341154d70a7b5a0b61eb7696d52
Parents: 2f6942f
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Dec 2 21:23:23 2016 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Dec 2 22:34:13 2016 -0500
----------------------------------------------------------------------
contrib/views/ambari-views-package/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3096c79d/contrib/views/ambari-views-package/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/ambari-views-package/pom.xml b/contrib/views/ambari-views-package/pom.xml
index cb070e6..16c9985 100644
--- a/contrib/views/ambari-views-package/pom.xml
+++ b/contrib/views/ambari-views-package/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.ambari.contrib.views</groupId>
<artifactId>ambari-contrib-views</artifactId>
- <version>2.5.0.0.0</version>
+ <version>2.0.0.0-SNAPSHOT</version>
</parent>
<description>
@@ -120,4 +120,4 @@
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
[30/50] [abbrv] ambari git commit: Revert "AMBARI-18662. Check that
Stack Version and repo URLs match up when registering repo (alexantonenko)"
Posted by nc...@apache.org.
Revert "AMBARI-18662. Check that Stack Version and repo URLs match up when registering repo (alexantonenko)"
This reverts commit 00f1e71b62f4133e70d7d8f9f2cd6fa0624ec40d.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2cd12ce4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2cd12ce4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2cd12ce4
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 2cd12ce4cebcf0a3a6fa58ff6f79a79b0a92f631
Parents: 149104c
Author: Alex Antonenko <hi...@gmail.com>
Authored: Fri Dec 2 20:22:01 2016 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Tue Dec 6 19:04:30 2016 +0200
----------------------------------------------------------------------
ambari-web/app/controllers/installer.js | 3 +--
.../app/controllers/wizard/step1_controller.js | 21 +-------------------
ambari-web/app/messages.js | 2 --
ambari-web/test/controllers/installer_test.js | 13 +++++-------
4 files changed, 7 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cd12ce4/ambari-web/app/controllers/installer.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/installer.js b/ambari-web/app/controllers/installer.js
index f736ba3..e2c60c4 100644
--- a/ambari-web/app/controllers/installer.js
+++ b/ambari-web/app/controllers/installer.js
@@ -841,7 +841,6 @@ App.InstallerController = App.WizardController.extend({
selectedStack.get('operatingSystems').forEach(function (os) {
if (os.get('isSelected') && !os.get('isEmpty')) {
os.get('repositories').forEach(function (repo) {
- if (!repo.get('isUtils') && wizardStep1Controller.inappropriateUrlForStackVersion(repo, stackVersion)) return;
repo.setProperties({
errorTitle: '',
errorContent: '',
@@ -889,8 +888,8 @@ App.InstallerController = App.WizardController.extend({
}
}
this.set('validationCnt', this.get('validationCnt') - 1);
- this.set('content.isCheckInProgress', false);
if (!this.get('validationCnt')) {
+ this.set('content.isCheckInProgress', false);
data.dfd.resolve();
}
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cd12ce4/ambari-web/app/controllers/wizard/step1_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/wizard/step1_controller.js b/ambari-web/app/controllers/wizard/step1_controller.js
index 72f3944..f397cd8 100644
--- a/ambari-web/app/controllers/wizard/step1_controller.js
+++ b/ambari-web/app/controllers/wizard/step1_controller.js
@@ -272,25 +272,6 @@ App.WizardStep1Controller = Em.Controller.extend({
this.selectStackBy('id', event.context.get('id'));
},
- inappropriateUrlForStackVersion: function (repo, stackVersion) {
- var baseUrl = repo.get('baseUrl'),
- splittedBaseUrlArray = baseUrl.split('/'),
- versionInUrl = baseUrl[baseUrl.length - 1] === '/' ? splittedBaseUrlArray[splittedBaseUrlArray.length - 2] :
- splittedBaseUrlArray[splittedBaseUrlArray.length - 1],
- versionInUrlString = String(parseFloat(versionInUrl)),
- UrlIsInvalidForStackVersion = versionInUrlString !== stackVersion;
-
- if (UrlIsInvalidForStackVersion) {
- repo.setProperties({
- validation: App.Repository.validation.INVALID,
- errorTitle: Em.I18n.t("installer.step1.error.inappropriateUrlForStackVersion.title"),
- errorContent: Em.I18n.t("installer.step1.error.inappropriateUrlForStackVersion.content").format(stackVersion, versionInUrlString)
- });
- }
-
- return UrlIsInvalidForStackVersion;
- },
-
/**
* Show popup with options to upload new version
*
@@ -461,7 +442,7 @@ App.WizardStep1Controller = Em.Controller.extend({
*/
enterUrlRadioButton: App.RadioButtonView.extend({
labelTranslate: 'installer.step1.useLocalRepo.enterUrl',
- checked: Em.computed.alias('controller.optionsToSelect.useLocalRepo.enterUrl.isSelected'),
+ checked: Em.computed.alias('controller.optionsToSelect.useLocalRepo.enterUrl.isSelected')
}),
click: function () {
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cd12ce4/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 090244c..595cd5d 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -642,8 +642,6 @@ Em.I18n.translations = {
'installer.step1.invalidURLAttention': '<b>Attention:</b> Please make sure all repository URLs are valid before proceeding.',
'installer.step1.checkAtLeastOneAttention': '<b>Attention:</b> Please check at least one repository.',
'installer.step1.retryRepoUrls': 'Click <b>here</b> to retry.',
- 'installer.step1.error.inappropriateUrlForStackVersion.title':'Inappropriate Url for selected Stack Version',
- 'installer.step1.error.inappropriateUrlForStackVersion.content': 'Your stack version is {0} when version in Url is {1}',
'installer.step2.header':'Install Options',
'installer.step2.body':'Enter the list of hosts to be included in the cluster and provide your SSH key.',
http://git-wip-us.apache.org/repos/asf/ambari/blob/2cd12ce4/ambari-web/test/controllers/installer_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/installer_test.js b/ambari-web/test/controllers/installer_test.js
index 63578bc..441af87 100644
--- a/ambari-web/test/controllers/installer_test.js
+++ b/ambari-web/test/controllers/installer_test.js
@@ -75,8 +75,8 @@ describe('App.InstallerController', function () {
Em.Object.create({
isSelected: true,
reload: false,
- id: 'nn-2.5',
- stackNameVersion: 'nn-2.5',
+ id: 'nn-cc',
+ stackNameVersion: 'nn-cc',
repositories: Em.A([
Em.Object.create({
isSelected: true,
@@ -89,7 +89,6 @@ describe('App.InstallerController', function () {
isEmpty: false,
repositories: Em.A([
Em.Object.create({
- baseUrl: 'url/2.5',
isEmpty: false,
errorTitle: '1',
errorContent: '1',
@@ -101,8 +100,7 @@ describe('App.InstallerController', function () {
})
]);
var wizard = Em.Object.create({
- skipValidationChecked: true,
- inappropriateUrlForStackVersion: App.WizardStep1Controller.create().inappropriateUrlForStackVersion
+ skipValidationChecked: true
});
it ('Should reload installed stacks', function() {
@@ -116,8 +114,8 @@ describe('App.InstallerController', function () {
{
"isSelected": true,
"reload": true,
- "id": "nn-2.5",
- "stackNameVersion": 'nn-2.5',
+ "id": "nn-cc",
+ "stackNameVersion": 'nn-cc',
"repositories": [
{
"isSelected": true,
@@ -130,7 +128,6 @@ describe('App.InstallerController', function () {
"isEmpty": false,
"repositories": [
{
- "baseUrl": "url/2.5",
"isEmpty": false,
"errorTitle": "",
"errorContent": "",
[06/50] [abbrv] ambari git commit: Revert "AMBARI-19055 - Removing
Tasks From host_role_command Causes Upgrades To Show As PENDING
(jonathanhurley)"
Posted by nc...@apache.org.
Revert "AMBARI-19055 - Removing Tasks From host_role_command Causes Upgrades To Show As PENDING (jonathanhurley)"
This reverts commit 32840c1ed434ae4cc99e3ab6f7d3f2604fb69d06.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5ec73bce
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5ec73bce
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5ec73bce
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 5ec73bce006a659f33c09c57cb50f28037eca5d7
Parents: 3096c79
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Sat Dec 3 08:22:31 2016 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Sat Dec 3 08:22:31 2016 -0500
----------------------------------------------------------------------
.../controller/internal/CalculatedStatus.java | 52 +++++---------------
.../internal/RequestResourceProvider.java | 34 +++++--------
.../internal/StageResourceProvider.java | 2 +-
.../internal/CalculatedStatusTest.java | 31 ------------
.../internal/RequestResourceProviderTest.java | 12 ++---
5 files changed, 28 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ec73bce/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
index 3c415df..3a86aef 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
@@ -53,17 +53,6 @@ public class CalculatedStatus {
*/
private final double percent;
- /**
- * A status which represents a COMPLETED state at 0%
- */
- public static final CalculatedStatus COMPLETED = new CalculatedStatus(HostRoleStatus.COMPLETED,
- HostRoleStatus.COMPLETED, 100.0);
-
- /**
- * A status which represents a PENDING state at 0%
- */
- public static final CalculatedStatus PENDING = new CalculatedStatus(HostRoleStatus.PENDING,
- HostRoleStatus.PENDING, 0.0);
// ----- Constructors ------------------------------------------------------
@@ -90,6 +79,12 @@ public class CalculatedStatus {
this.percent = percent;
}
+ /**
+ * Static factory method to get Status that represents a Completed state
+ */
+ public static CalculatedStatus getCompletedStatus() {
+ return new CalculatedStatus(HostRoleStatus.COMPLETED, HostRoleStatus.COMPLETED, 100.0);
+ }
// ----- CalculatedStatus --------------------------------------------------
@@ -296,25 +291,14 @@ public class CalculatedStatus {
}
/**
- * Calculates the overall status of an upgrade. If there are no tasks, then a
- * status of {@link HostRoleStatus#COMPLETED} is returned.
- *
- * @param stageDto
- * the map of stage-to-summary value objects
- * @param stageIds
- * the stage ids to consider from the value objects
+ * Calculates the overall status of an upgrade.
+ * @param stageDto the map of stage-to-summary value objects
+ * @param stageIds the stage ids to consider from the value objects
* @return the calculated status
*/
public static CalculatedStatus statusFromStageSummary(Map<Long, HostRoleCommandStatusSummaryDTO> stageDto,
Set<Long> stageIds) {
- // if either are empty, then we have no tasks and therefore no status - we
- // should return COMPLETED. This can happen if someone removes all tasks but
- // leaves the stages and request
- if (stageDto.isEmpty() || stageIds.isEmpty()) {
- return COMPLETED;
- }
-
Collection<HostRoleStatus> stageStatuses = new HashSet<>();
Collection<HostRoleStatus> stageDisplayStatuses = new HashSet<>();
Collection<HostRoleStatus> taskStatuses = new ArrayList<>();
@@ -394,28 +378,19 @@ public class CalculatedStatus {
*/
public static HostRoleStatus calculateSummaryStatusOfStage(Map<HostRoleStatus, Integer> counters,
int total, boolean skippable) {
-
- // when there are 0 tasks, return COMPLETED
- if (total == 0) {
- return HostRoleStatus.COMPLETED;
- }
-
if (counters.get(HostRoleStatus.PENDING) == total) {
return HostRoleStatus.PENDING;
}
-
// By definition, any tasks in a future stage must be held in a PENDING status.
if (counters.get(HostRoleStatus.HOLDING) > 0 || counters.get(HostRoleStatus.HOLDING_FAILED) > 0 || counters.get(HostRoleStatus.HOLDING_TIMEDOUT) > 0) {
return counters.get(HostRoleStatus.HOLDING) > 0 ? HostRoleStatus.HOLDING :
counters.get(HostRoleStatus.HOLDING_FAILED) > 0 ? HostRoleStatus.HOLDING_FAILED :
HostRoleStatus.HOLDING_TIMEDOUT;
}
-
// Because tasks are not skippable, guaranteed to be FAILED
if (counters.get(HostRoleStatus.FAILED) > 0 && !skippable) {
return HostRoleStatus.FAILED;
}
-
// Because tasks are not skippable, guaranteed to be TIMEDOUT
if (counters.get(HostRoleStatus.TIMEDOUT) > 0 && !skippable) {
return HostRoleStatus.TIMEDOUT;
@@ -426,11 +401,9 @@ public class CalculatedStatus {
if (counters.get(HostRoleStatus.ABORTED) > 0 && numActiveTasks == 0) {
return HostRoleStatus.ABORTED;
}
-
if (counters.get(HostRoleStatus.COMPLETED) == total) {
return HostRoleStatus.COMPLETED;
}
-
return HostRoleStatus.IN_PROGRESS;
}
@@ -442,8 +415,7 @@ public class CalculatedStatus {
*
* @return summary request status based on statuses of tasks in different states.
*/
- protected static HostRoleStatus calculateSummaryStatusOfUpgrade(
- Map<HostRoleStatus, Integer> counters, int total) {
+ private static HostRoleStatus calculateSummaryStatusOfUpgrade(Map<HostRoleStatus, Integer> counters, int total) {
return calculateSummaryStatusOfStage(counters, total, false);
}
@@ -456,8 +428,8 @@ public class CalculatedStatus {
*
* @return summary request status based on statuses of tasks in different states.
*/
- protected static HostRoleStatus calculateSummaryDisplayStatus(
- Map<HostRoleStatus, Integer> counters, int total, boolean skippable) {
+ private static HostRoleStatus calculateSummaryDisplayStatus(Map<HostRoleStatus, Integer> counters,
+ int total, boolean skippable) {
return counters.get(HostRoleStatus.SKIPPED_FAILED) > 0 ? HostRoleStatus.SKIPPED_FAILED :
counters.get(HostRoleStatus.FAILED) > 0 ? HostRoleStatus.FAILED:
calculateSummaryStatusOfStage(counters, total, skippable);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ec73bce/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
index 0690ee7..8c1bc57 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
@@ -69,10 +69,10 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.topology.LogicalRequest;
import org.apache.ambari.server.topology.TopologyManager;
-import org.apache.commons.lang.StringUtils;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
+import org.apache.commons.lang.StringUtils;
/**
* Resource provider for request resources.
@@ -566,18 +566,13 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider
@Override
public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
ServiceComponentTuple that = (ServiceComponentTuple) o;
- if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) {
+ if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null)
return false;
- }
return !(componentName != null ? !componentName.equals(that.componentName) : that.componentName != null);
}
@@ -744,20 +739,15 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider
// get summaries from TopologyManager for logical requests
summary.putAll(topologyManager.getStageSummaries(entity.getRequestId()));
- // summary might be empty due to delete host have cleared all
- // HostRoleCommands or due to hosts haven't registered yet with the cluster
- // when the cluster is provisioned with a Blueprint
- final CalculatedStatus status;
LogicalRequest logicalRequest = topologyManager.getRequest(entity.getRequestId());
- if (summary.isEmpty() && null != logicalRequest) {
- // in this case, it appears that there are no tasks but this is a logical
- // topology request, so it's a matter of hosts simply not registering yet
- // for tasks to be created
- status = CalculatedStatus.PENDING;
- } else {
- // there are either tasks or this is not a logical request, so do normal
- // status calculations
- status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
+
+ CalculatedStatus status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
+ if (summary.isEmpty() && logicalRequest == null) {
+
+ // summary might be empty due to delete host have cleared all HostRoleCommands
+ // or due to hosts haven't registered yet with the cluster when the cluster is provisioned
+ // with a Blueprint
+ status = CalculatedStatus.getCompletedStatus();
}
setResourceProperty(resource, REQUEST_STATUS_PROPERTY_ID, status.getStatus().toString(), requestedPropertyIds);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ec73bce/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
index a778882..59dd9d9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
@@ -339,7 +339,7 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
CalculatedStatus status;
if (summary.isEmpty()) {
// Delete host might have cleared all HostRoleCommands
- status = CalculatedStatus.COMPLETED;
+ status = CalculatedStatus.getCompletedStatus();
} else {
status = CalculatedStatus.statusFromStageSummary(summary, Collections.singleton(entity.getStageId()));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ec73bce/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
index a96f395..6f592cd 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
@@ -657,37 +657,6 @@ public class CalculatedStatusTest {
assertEquals(HostRoleStatus.IN_PROGRESS, calc.getStatus());
}
- /**
- * Tests that when there are no tasks and all counts are 0, that the returned
- * status is {@link HostRoleStatus#COMPLETED}.
- *
- * @throws Exception
- */
- @Test
- public void testGetCompletedStatusForNoTasks() throws Exception {
- // no status / no tasks
- CalculatedStatus status = CalculatedStatus.statusFromTaskEntities(
- new ArrayList<HostRoleCommandEntity>(), false);
-
- assertEquals(HostRoleStatus.COMPLETED, status.getStatus());
-
- // empty summaries
- status = CalculatedStatus.statusFromStageSummary(
- new HashMap<Long, HostRoleCommandStatusSummaryDTO>(), new HashSet<Long>());
-
- assertEquals(HostRoleStatus.COMPLETED, status.getStatus());
-
- // generate a map of 0's - COMPLETED=0, IN_PROGRESS=0, etc
- Map<HostRoleStatus, Integer> counts = CalculatedStatus.calculateStatusCounts(new ArrayList<HostRoleStatus>());
- Map<HostRoleStatus, Integer> displayCounts = CalculatedStatus.calculateStatusCounts(new ArrayList<HostRoleStatus>());
-
- HostRoleStatus hostRoleStatus = CalculatedStatus.calculateSummaryStatusOfUpgrade(counts, 0);
- HostRoleStatus hostRoleDisplayStatus = CalculatedStatus.calculateSummaryDisplayStatus(displayCounts, 0, false);
-
- assertEquals(HostRoleStatus.COMPLETED, hostRoleStatus);
- assertEquals(HostRoleStatus.COMPLETED, hostRoleDisplayStatus);
- }
-
private Collection<HostRoleCommandEntity> getTaskEntities(HostRoleStatus... statuses) {
Collection<HostRoleCommandEntity> entities = new LinkedList<HostRoleCommandEntity>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ec73bce/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
index f7dff11..5dfc74d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
@@ -20,10 +20,11 @@ package org.apache.ambari.server.controller.internal;
import static org.apache.ambari.server.controller.internal.HostComponentResourceProvider.HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID;
+import org.apache.ambari.server.topology.Blueprint;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.newCapture;
import static org.powermock.api.easymock.PowerMock.createMock;
import static org.powermock.api.easymock.PowerMock.createNiceMock;
@@ -76,7 +77,6 @@ import org.apache.ambari.server.security.authorization.AuthorizationHelperInitia
import org.apache.ambari.server.security.authorization.RoleAuthorization;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.HostGroupInfo;
import org.apache.ambari.server.topology.LogicalRequest;
@@ -1200,7 +1200,7 @@ public class RequestResourceProviderTest {
}
}
Assert.assertNotNull(propertyIdToAssert);
- Assert.assertEquals("true", propertyValueToAssert);
+ Assert.assertEquals("true", (String) propertyValueToAssert);
}
@Test
@@ -1624,12 +1624,6 @@ public class RequestResourceProviderTest {
verify(managementController, actionManager, clusters, requestMock, requestDAO, hrcDAO);
}
- /**
- * Tests that topology requests return different status (PENDING) if there are
- * no tasks. Normal requests should return COMPLETED.
- *
- * @throws Exception
- */
@Test
@PrepareForTest(AmbariServer.class)
public void testGetLogicalRequestStatusWithNoTasks() throws Exception {
[32/50] [abbrv] ambari git commit: AMBARI-19112 : Grafana start
failing at create datasource step (and a minor AbstractUpgradeCatalog log
line change).
Posted by nc...@apache.org.
AMBARI-19112 : Grafana start failing at create datasource step (and a minor AbstractUpgradeCatalog log line change).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/502d4447
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/502d4447
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/502d4447
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 502d4447fa7f78421e606cff83661b41c4923a61
Parents: 9b46fd2
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Dec 6 12:07:38 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Dec 6 12:07:48 2016 -0800
----------------------------------------------------------------------
.../apache/ambari/server/upgrade/AbstractUpgradeCatalog.java | 4 ++--
.../0.1.0/package/scripts/metrics_grafana_util.py | 5 +++--
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/502d4447/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index 7f91a68..4f6cbb7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -585,8 +585,8 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
}
if (!Maps.difference(oldConfigProperties, mergedProperties).areEqual()) {
- LOG.info("Applying configuration with tag '{}' to " +
- "cluster '{}'", newTag, cluster.getClusterName());
+ LOG.info("Applying configuration with tag '{}' and configType '{}' to " +
+ "cluster '{}'", newTag, configType, cluster.getClusterName());
Map<String, Map<String, String>> propertiesAttributes = null;
if (oldConfig != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/502d4447/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
index 6030fc4..41bd2b4 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -34,8 +34,8 @@ import ambari_simplejson as json
import network
import os
-GRAFANA_CONNECT_TRIES = 5
-GRAFANA_CONNECT_TIMEOUT = 10
+GRAFANA_CONNECT_TRIES = 15
+GRAFANA_CONNECT_TIMEOUT = 20
GRAFANA_SEARCH_BUILTIN_DASHBOARDS = "/api/search?tag=builtin"
GRAFANA_DATASOURCE_URL = "/api/datasources"
GRAFANA_DASHBOARDS_URL = "/api/dashboards/db"
@@ -202,6 +202,7 @@ def do_ams_collector_post(metric_collector_host, params):
post_metrics_to_collector(ams_metrics_post_url, metric_collector_host, params.metric_collector_port, params.metric_collector_https_enabled,
metric_json, headers, ca_certs)
+
def create_ams_datasource():
import params
server = Server(protocol = params.ams_grafana_protocol.strip(),
[42/50] [abbrv] ambari git commit: AMBARI-19099. Improve and Fix
'Wizard pages' after new guidelines. (xiwang)
Posted by nc...@apache.org.
AMBARI-19099. Improve and Fix 'Wizard pages' after new guidelines. (xiwang)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1e261db0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1e261db0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1e261db0
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 1e261db06128143a4e631aff899704cf12709413
Parents: a62119a
Author: Xi Wang <xi...@apache.org>
Authored: Mon Dec 5 16:01:52 2016 -0800
Committer: Xi Wang <xi...@apache.org>
Committed: Tue Dec 6 17:25:21 2016 -0800
----------------------------------------------------------------------
.../app/styles/theme/bootstrap-ambari.css | 25 +++++++++++---------
ambari-web/app/styles/wizard.less | 13 ++++++++--
ambari-web/app/templates/wizard/step8.hbs | 2 +-
3 files changed, 26 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e261db0/ambari-web/app/styles/theme/bootstrap-ambari.css
----------------------------------------------------------------------
diff --git a/ambari-web/app/styles/theme/bootstrap-ambari.css b/ambari-web/app/styles/theme/bootstrap-ambari.css
index 738fa59..5a641b3 100644
--- a/ambari-web/app/styles/theme/bootstrap-ambari.css
+++ b/ambari-web/app/styles/theme/bootstrap-ambari.css
@@ -510,19 +510,15 @@ h2.table-title {
margin: 15px 20px;
}
.wizard .wizard-body {
- background: #ebecf1;
- /* Old browsers */
- background: -moz-linear-gradient(to right, #323544, #323544 25%, #ebecf1 25%, #ebecf1);
- /* FF3.6-15 */
- background: -webkit-linear-gradient(to right, #323544, #323544 25%, #ebecf1 25%, #ebecf1);
- /* Chrome10-25,Safari5.1-6 */
- background: linear-gradient(to right, #323544, #323544 25%, #ebecf1 25%, #ebecf1);
- /* W3C, IE10+, FF16+, Chrome26+, Opera12+, Safari7+ */
+ overflow: hidden;
margin: 0;
}
.wizard .wizard-body .wizard-content {
background: #ebecf1;
padding-top: 25px;
+ float: left;
+ margin-bottom: -99999px;
+ padding-bottom: 99999px;
}
.wizard .wizard-body .wizard-content .step-title {
font-family: 'Roboto', sans-serif;
@@ -554,6 +550,9 @@ h2.table-title {
min-height: 650px;
padding-top: 25px;
background-color: #323544;
+ float: left;
+ margin-bottom: -99999px;
+ padding-bottom: 99999px;
}
.wizard .wizard-body .wizard-nav .nav li {
padding: 10px 0;
@@ -588,6 +587,9 @@ h2.table-title {
color: #bbbbbb;
margin-left: 30px;
}
+.wizard .wizard-body .wizard-nav .nav li .step-index {
+ line-height: 18px;
+}
.wizard .wizard-body .wizard-nav .nav li .step-description {
font-family: 'Roboto', sans-serif;
font-weight: normal;
@@ -602,7 +604,7 @@ h2.table-title {
background-color: #1EB475;
color: white;
font-size: 10px;
- padding-left: 3px;
+ padding-left: 2px;
}
.wizard .wizard-body .wizard-nav .nav li.completed .step-marker .step-index {
display: none;
@@ -634,8 +636,9 @@ h2.table-title {
border-color: #bbbbbb;
}
.wizard .wizard-body .wizard-nav .nav li.disabled.completed .step-marker {
- background-color: #bbbbbb;
- color: #323544;
+ background-color: #1EB475;
+ border: 2px solid #1EB475;
+ color: white;
}
.wizard .wizard-body .wizard-nav .nav-pills > li.active > a,
.wizard .wizard-body .wizard-nav .nav-pills > li.active > a:focus,
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e261db0/ambari-web/app/styles/wizard.less
----------------------------------------------------------------------
diff --git a/ambari-web/app/styles/wizard.less b/ambari-web/app/styles/wizard.less
index 57dae45..9920319 100644
--- a/ambari-web/app/styles/wizard.less
+++ b/ambari-web/app/styles/wizard.less
@@ -36,7 +36,6 @@
padding-right: 15px;
}
.btn-area {
- margin-top: 20px;
.btn.btn-info {
margin-right: 10px;
}
@@ -433,7 +432,7 @@
color: #fff;
cursor: pointer;
font-weight: bold;
- text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+ line-height: 16px;
&:hover {
background-color: @green;
}
@@ -471,6 +470,12 @@
ul {
li {
list-style: none;
+ span.text {
+ line-height: 1.5;
+ }
+ b.service-name {
+ line-height: 2;
+ }
}
}
max-height: 440px;
@@ -490,6 +495,10 @@
ul {
li {
list-style: none;
+ span.text-info,
+ span.text-success {
+ line-height: 1.5;
+ }
}
}
max-height: 440px;
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e261db0/ambari-web/app/templates/wizard/step8.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/wizard/step8.hbs b/ambari-web/app/templates/wizard/step8.hbs
index 5c2e8e6..9e50e30 100644
--- a/ambari-web/app/templates/wizard/step8.hbs
+++ b/ambari-web/app/templates/wizard/step8.hbs
@@ -63,7 +63,7 @@
<ul>
{{#each controller.services}}
<li>
- <em><b>{{display_name}}</b></em>
+ <em><b class="service-name">{{display_name}}</b></em>
<ul>
{{#each component in this.service_components}}
<li>
[16/50] [abbrv] ambari git commit: AMBARI-18065. Change Ranger Admin
start script to setup db for ranger (magyari_sandor)
Posted by nc...@apache.org.
AMBARI-18065. Change Ranger Admin start script to setup db for ranger (magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f5051dcb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f5051dcb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f5051dcb
Branch: refs/heads/branch-dev-patch-upgrade
Commit: f5051dcb7f17f0e26fed9e70eef35a3758a35e60
Parents: 431d265
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Thu Dec 1 19:36:01 2016 +0100
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Mon Dec 5 16:39:13 2016 +0100
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../RANGER/0.4.0/package/scripts/params.py | 1 +
.../0.4.0/package/scripts/ranger_admin.py | 36 +-
.../HDP/2.0.6/properties/stack_features.json | 5 +
.../stacks/2.6/RANGER/test_ranger_admin.py | 504 ++++++++++++
.../2.6/configs/ranger-admin-default.json | 704 +++++++++++++++++
.../2.6/configs/ranger-admin-secured.json | 773 +++++++++++++++++++
7 files changed, 2010 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 3431495..46562e0 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -97,6 +97,7 @@ class StackFeature:
RANGER_PID_SUPPORT = "ranger_pid_support"
RANGER_KMS_PID_SUPPORT = "ranger_kms_pid_support"
RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change"
+ RANGER_SETUP_DB_ON_START = "ranger_setup_db_on_start"
STORM_METRICS_APACHE_CLASSES = "storm_metrics_apache_classes"
SPARK_JAVA_OPTS_SUPPORT = "spark_java_opts_support"
ATLAS_HBASE_SETUP = "atlas_hbase_setup"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
index e27b363..f2cc940 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
@@ -70,6 +70,7 @@ stack_supports_usersync_passwd = check_stack_feature(StackFeature.RANGER_USERSYN
stack_supports_infra_client = check_stack_feature(StackFeature.RANGER_INSTALL_INFRA_CLIENT, version_for_stack_feature_checks)
stack_supports_pid = check_stack_feature(StackFeature.RANGER_PID_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_admin_password_change = check_stack_feature(StackFeature.RANGER_ADMIN_PASSWD_CHANGE, version_for_stack_feature_checks)
+stack_supports_ranger_setup_db_on_start = check_stack_feature(StackFeature.RANGER_SETUP_DB_ON_START, version_for_stack_feature_checks)
downgrade_from_version = default("/commandParams/downgrade_from_version", None)
upgrade_direction = default("/commandParams/upgrade_direction", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
index a9656c7..b849d58 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
@@ -45,18 +45,9 @@ class RangerAdmin(Script):
self.install_packages(env)
import params
env.set_params(params)
- if params.xml_configurations_supported:
- from setup_ranger_xml import setup_ranger_db
- setup_ranger_db()
-
- self.configure(env)
-
- if params.xml_configurations_supported:
- from setup_ranger_xml import setup_java_patch
- setup_java_patch()
-
- if params.stack_supports_ranger_admin_password_change:
- setup_ranger_admin_passwd_change()
+ # call config and setup db only in case of HDP version < 2.6
+ if not params.stack_supports_ranger_setup_db_on_start:
+ self.configure(env, setup_db=True)
def stop(self, env, upgrade_type=None):
import params
@@ -93,7 +84,9 @@ class RangerAdmin(Script):
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
- self.configure(env, upgrade_type=upgrade_type)
+
+ # setup db only if in case HDP version is > 2.6
+ self.configure(env, upgrade_type=upgrade_type, setup_db=params.stack_supports_ranger_setup_db_on_start)
if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
solr_cloud_util.setup_solr_client(params.config, custom_log4j = params.custom_log4j)
@@ -122,7 +115,7 @@ class RangerAdmin(Script):
raise ComponentIsNotRunning()
pass
- def configure(self, env, upgrade_type=None):
+ def configure(self, env, upgrade_type=None, setup_db=False):
import params
env.set_params(params)
if params.xml_configurations_supported:
@@ -130,8 +123,23 @@ class RangerAdmin(Script):
else:
from setup_ranger import ranger
+ # set up db if we are not upgrading and setup_db is true
+ if setup_db and upgrade_type is None:
+ if params.xml_configurations_supported:
+ from setup_ranger_xml import setup_ranger_db
+ setup_ranger_db()
+
ranger('ranger_admin', upgrade_type=upgrade_type)
+ # set up java patches if we are not upgrading and setup_db is true
+ if setup_db and upgrade_type is None:
+ if params.xml_configurations_supported:
+ from setup_ranger_xml import setup_java_patch
+ setup_java_patch()
+
+ if params.stack_supports_ranger_admin_password_change:
+ setup_ranger_admin_passwd_change()
+
def set_ru_rangeradmin_in_progress(self, upgrade_marker_file):
config_dir = os.path.dirname(upgrade_marker_file)
try:
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index dd87b72..27a755c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -289,6 +289,11 @@
"min_version": "2.5.0.0"
},
{
+ "name": "ranger_setup_db_on_start",
+ "description": "Allows setup of ranger db and java patches to be called multiple times on each START",
+ "min_version": "2.6.0.0"
+ },
+ {
"name": "storm_metrics_apache_classes",
"description": "Metrics sink for Storm that uses Apache class names",
"min_version": "2.5.0.0"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py b/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
new file mode 100644
index 0000000..a3ce186
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/RANGER/test_ranger_admin.py
@@ -0,0 +1,504 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+from mock.mock import MagicMock, patch
+from stacks.utils.RMFTestCase import *
+from only_for_platform import not_for_platform, PLATFORM_WINDOWS
+
+@not_for_platform(PLATFORM_WINDOWS)
+class TestRangerAdmin(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "RANGER/0.4.0/package"
+ STACK_VERSION = "2.6"
+
+ @patch("os.path.isfile")
+ def test_start_default(self, isfile_mock):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_admin.py",
+ classname = "RangerAdmin",
+ command = "start",
+ config_file="ranger-admin-default.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default()
+
+ self.assertResourceCalled('Directory', '/var/log/ambari-infra-solr-client',
+ create_parents = True,
+ mode = 0755,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-infra-solr-client',
+ cd_access = 'a',
+ create_parents = True,
+ mode = 0755,
+ recursive_ownership = True,
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/solrCloudCli.sh',
+ content = StaticFile('/usr/lib/ambari-infra-solr-client/solrCloudCli.sh'),
+ mode = 0755,
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['infra-solr-client-log4j']['content']),
+ mode = 0644,
+ )
+ self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
+ content = '',
+ mode = 0664,
+ )
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_ranger_audits_0.[0-9]*',
+ action=['delete'],
+ create_parents=True)
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+
+ self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ps -ef | grep proc_rangeradmin | grep -v grep',
+ user = 'ranger',
+ )
+
+ self.assertTrue(isfile_mock.called)
+ self.assertNoMoreResources()
+
+ @patch("os.path.isfile")
+ def test_start_secured(self, isfile_mock):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/ranger_admin.py",
+ classname = "RangerAdmin",
+ command = "start",
+ config_file="ranger-admin-secured.json",
+ stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured()
+
+ self.assertResourceCalled('Directory', '/var/log/ambari-infra-solr-client',
+ create_parents = True,
+ mode = 0755,
+ cd_access = 'a',
+ )
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-infra-solr-client',
+ cd_access = 'a',
+ create_parents = True,
+ mode = 0755,
+ recursive_ownership = True,
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/solrCloudCli.sh',
+ content = StaticFile('/usr/lib/ambari-infra-solr-client/solrCloudCli.sh'),
+ mode = 0755,
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['infra-solr-client-log4j']['content']),
+ mode = 0644,
+ )
+ self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
+ content = '',
+ mode = 0664,
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jaas.conf',
+ content = Template('ranger_solr_jaas_conf.j2'),
+ owner = 'ranger',
+ )
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /ambari-solr --check-znode --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/solr_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_ranger_audits_0.[0-9]*',
+ action=['delete'],
+ create_parents=True)
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+
+ self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ps -ef | grep proc_rangeradmin | grep -v grep',
+ user = 'ranger',
+ )
+
+ self.assertTrue(isfile_mock.called)
+ self.assertNoMoreResources()
+
+
+ def assert_setup_db(self):
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java-old.jar',
+ action = ['delete'],
+ )
+
+ self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar',
+ '/usr/hdp/2.6.0.0-801/ranger-admin/ews/lib'),
+ sudo = True,
+ path = ['/bin', '/usr/bin/']
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/2.6.0.0-801/ranger-admin/ews/lib/mysql-connector-java.jar',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/2.6.0.0-801/ranger-admin/install.properties',
+ properties = self.getConfig()['configurations']['admin-properties'],
+ owner = 'ranger'
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/2.6.0.0-801/ranger-admin/install.properties',
+ owner = 'ranger',
+ properties = {'SQL_CONNECTOR_JAR':
+ '/usr/hdp/2.6.0.0-801/ranger-admin/ews/lib/mysql-connector-java.jar'}
+ )
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-admin/install.properties',
+ owner = 'ranger',
+ properties = {'audit_store': 'solr'}
+ )
+
+ self.assertResourceCalled('Execute', ('ambari-python-wrap /usr/hdp/current/ranger-admin/dba_script.py -q'),
+ user = 'ranger',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45',
+ 'RANGER_ADMIN_HOME': u'/usr/hdp/current/ranger-admin'},
+ logoutput = True
+ )
+
+ self.assertResourceCalled('Execute', ('ambari-python-wrap /usr/hdp/current/ranger-admin/db_setup.py'),
+ user = 'ranger',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45',
+ 'RANGER_ADMIN_HOME': u'/usr/hdp/current/ranger-admin'},
+ logoutput = True
+ )
+
+ def assert_configure_default(self):
+
+ ### assert db setup
+ self.assert_setup_db()
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-admin/conf',
+ owner = 'ranger',
+ group = 'ranger',
+ create_parents = True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java-old.jar',
+ action = ['delete'],
+ )
+
+ self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar', '/usr/hdp/current/ranger-admin/ews/lib'),
+ sudo = True,
+ path = ['/bin', '/usr/bin/']
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java.jar',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-admin/install.properties',
+ properties = self.getConfig()['configurations']['admin-properties'],
+ owner = 'ranger'
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-admin/install.properties',
+ owner = 'ranger',
+ properties = {'SQL_CONNECTOR_JAR': '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java.jar'}
+ )
+
+ self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute',
+ '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java.jar:/usr/hdp/current/ranger-admin/ews/lib/* org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6401.ambari.apache.org:3306/ranger01\' rangeradmin01 rangeradmin01 com.mysql.jdbc.Driver',
+ path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
+ tries=5,
+ try_sleep=10,
+ environment = {}
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-admin/ews/webapp/WEB-INF/classes/conf', '/usr/hdp/current/ranger-admin/conf'),
+ not_if = 'ls /usr/hdp/current/ranger-admin/conf',
+ only_if = 'ls /usr/hdp/current/ranger-admin/ews/webapp/WEB-INF/classes/conf',
+ sudo = True
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-admin/',
+ owner='ranger',
+ group='ranger',
+ recursive_ownership = True
+ )
+
+ self.assertResourceCalled('Directory', '/var/run/ranger',
+ mode=0755,
+ owner = 'ranger',
+ group = 'hadoop',
+ cd_access = "a",
+ create_parents=True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger-admin-env-piddir.sh',
+ content = 'export RANGER_PID_DIR_PATH=/var/run/ranger\nexport RANGER_USER=ranger',
+ owner = 'ranger',
+ group = 'ranger',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ranger/admin',
+ owner='ranger',
+ group='ranger',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger-admin-env-logdir.sh',
+ content = 'export RANGER_ADMIN_LOG_DIR=/var/log/ranger/admin',
+ owner = 'ranger',
+ group = 'ranger',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger-admin-default-site.xml',
+ owner = 'ranger',
+ group = 'ranger'
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/security-applicationContext.xml',
+ owner = 'ranger',
+ group = 'ranger'
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-admin/ews/ranger-admin-services.sh', '/usr/bin/ranger-admin'),
+ not_if = 'ls /usr/bin/ranger-admin',
+ only_if = 'ls /usr/hdp/current/ranger-admin/ews/ranger-admin-services.sh',
+ sudo = True
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-admin-site.xml',
+ owner = 'ranger',
+ group = 'ranger',
+ conf_dir = '/usr/hdp/current/ranger-admin/conf',
+ configurations = self.getConfig()['configurations']['ranger-admin-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-admin-site'],
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-admin/conf/ranger_jaas',
+ owner ='ranger',
+ group ='ranger',
+ mode = 0700
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/webapp/WEB-INF/log4j.properties',
+ owner = 'ranger',
+ group = 'ranger',
+ content = self.getConfig()['configurations']['admin-log4j']['content'],
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/jdk64/jdk1.7.0_45/bin/java', '-cp', '/usr/hdp/current/ranger-admin/cred/lib/*', 'org.apache.ranger.credentialapi.buildks', 'create', 'rangeradmin', '-value', 'rangeradmin01', '-provider', 'jceks://file/etc/ranger/admin/rangeradmin.jceks'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo = True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/admin/rangeradmin.jceks',
+ owner = 'ranger',
+ group = 'ranger',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ owner = 'ranger',
+ group = 'ranger',
+ conf_dir = '/usr/hdp/current/ranger-admin/conf',
+ configurations = self.getConfig()['configurations']['core-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('ambari-python-wrap /usr/hdp/current/ranger-admin/db_setup.py -javapatch'),
+ user = 'ranger',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45',
+ 'RANGER_ADMIN_HOME': u'/usr/hdp/current/ranger-admin'},
+ logoutput = True
+ )
+
+ def assert_configure_secured(self):
+
+ ### assert db setup
+ self.assert_setup_db()
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-admin/conf',
+ owner = 'ranger',
+ group = 'ranger',
+ create_parents = True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java-old.jar',
+ action = ['delete'],
+ )
+
+ self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar', '/usr/hdp/current/ranger-admin/ews/lib'),
+ sudo = True,
+ path = ['/bin', '/usr/bin/']
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java.jar',
+ mode = 0644
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-admin/install.properties',
+ properties = self.getConfig()['configurations']['admin-properties'],
+ owner = 'ranger'
+ )
+
+ self.assertResourceCalled('ModifyPropertiesFile', '/usr/hdp/current/ranger-admin/install.properties',
+ owner = 'ranger',
+ properties = {'SQL_CONNECTOR_JAR': '/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java.jar'}
+ )
+
+ self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute',
+ '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/ranger-admin/ews/lib/mysql-connector-java.jar:/usr/hdp/current/ranger-admin/ews/lib/* org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6401.ambari.apache.org:3306/ranger01\' rangeradmin01 rangeradmin01 com.mysql.jdbc.Driver',
+ path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
+ tries=5,
+ try_sleep=10,
+ environment = {}
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-admin/ews/webapp/WEB-INF/classes/conf', '/usr/hdp/current/ranger-admin/conf'),
+ not_if = 'ls /usr/hdp/current/ranger-admin/conf',
+ only_if = 'ls /usr/hdp/current/ranger-admin/ews/webapp/WEB-INF/classes/conf',
+ sudo = True
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-admin/',
+ owner='ranger',
+ group='ranger',
+ recursive_ownership = True
+ )
+
+ self.assertResourceCalled('Directory', '/var/run/ranger',
+ mode=0755,
+ owner = 'ranger',
+ group = 'hadoop',
+ cd_access = "a",
+ create_parents=True
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger-admin-env-piddir.sh',
+ content = 'export RANGER_PID_DIR_PATH=/var/run/ranger\nexport RANGER_USER=ranger',
+ owner = 'ranger',
+ group = 'ranger',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ranger/admin',
+ owner='ranger',
+ group='ranger',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger-admin-env-logdir.sh',
+ content = 'export RANGER_ADMIN_LOG_DIR=/var/log/ranger/admin',
+ owner = 'ranger',
+ group = 'ranger',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger-admin-default-site.xml',
+ owner = 'ranger',
+ group = 'ranger'
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/security-applicationContext.xml',
+ owner = 'ranger',
+ group = 'ranger'
+ )
+
+ self.assertResourceCalled('Execute', ('ln', '-sf', '/usr/hdp/current/ranger-admin/ews/ranger-admin-services.sh', '/usr/bin/ranger-admin'),
+ not_if = 'ls /usr/bin/ranger-admin',
+ only_if = 'ls /usr/hdp/current/ranger-admin/ews/ranger-admin-services.sh',
+ sudo = True
+ )
+
+ self.assertResourceCalled('XmlConfig', 'ranger-admin-site.xml',
+ owner = 'ranger',
+ group = 'ranger',
+ conf_dir = '/usr/hdp/current/ranger-admin/conf',
+ configurations = self.getConfig()['configurations']['ranger-admin-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['ranger-admin-site'],
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Directory', '/usr/hdp/current/ranger-admin/conf/ranger_jaas',
+ owner ='ranger',
+ group ='ranger',
+ mode = 0700
+ )
+
+ self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/ews/webapp/WEB-INF/log4j.properties',
+ owner = 'ranger',
+ group = 'ranger',
+ content = self.getConfig()['configurations']['admin-log4j']['content'],
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('/usr/jdk64/jdk1.7.0_45/bin/java', '-cp', '/usr/hdp/current/ranger-admin/cred/lib/*', 'org.apache.ranger.credentialapi.buildks', 'create', 'rangeradmin', '-value', 'rangeradmin01', '-provider', 'jceks://file/etc/ranger/admin/rangeradmin.jceks'),
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ logoutput=True,
+ sudo = True
+ )
+
+ self.assertResourceCalled('File', '/etc/ranger/admin/rangeradmin.jceks',
+ owner = 'ranger',
+ group = 'ranger',
+ mode = 0640
+ )
+
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ owner = 'ranger',
+ group = 'ranger',
+ conf_dir = '/usr/hdp/current/ranger-admin/conf',
+ configurations = self.getConfig()['configurations']['core-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+ mode = 0644
+ )
+
+ self.assertResourceCalled('Execute', ('ambari-python-wrap /usr/hdp/current/ranger-admin/db_setup.py -javapatch'),
+ user = 'ranger',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45',
+ 'RANGER_ADMIN_HOME': u'/usr/hdp/current/ranger-admin'},
+ logoutput = True
+ )
\ No newline at end of file
[22/50] [abbrv] ambari git commit: AMBARI-19082. Ambari Server
upgrade from 2.0.2 to 2.2.2 changed the following configs and added a port of
-1.(vbrodetskyi)
Posted by nc...@apache.org.
AMBARI-19082. Ambari Server upgrade from 2.0.2 to 2.2.2 changed the following configs and added a port of -1.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/edf8ab24
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/edf8ab24
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/edf8ab24
Branch: refs/heads/branch-dev-patch-upgrade
Commit: edf8ab2432af6b73f73c4ae386ff13c93c6c38c2
Parents: a00ffff
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Dec 6 00:06:31 2016 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Dec 6 00:06:31 2016 +0200
----------------------------------------------------------------------
.../server/upgrade/AbstractUpgradeCatalog.java | 21 ++++++++++++--------
.../server/upgrade/UpgradeCatalog210Test.java | 2 ++
2 files changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/edf8ab24/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index ce9fcc6..7f91a68 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -17,6 +17,11 @@
*/
package org.apache.ambari.server.upgrade;
+import javax.persistence.EntityManager;
+import javax.xml.bind.JAXBException;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
@@ -42,11 +47,6 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.persistence.EntityManager;
-import javax.xml.bind.JAXBException;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
@@ -447,11 +447,16 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
Config hdfsSiteConfig = cluster.getDesiredConfigByType(CONFIGURATION_TYPE_HDFS_SITE);
if (hdfsSiteConfig != null) {
Map<String, String> properties = hdfsSiteConfig.getProperties();
+ if (properties.containsKey("dfs.internal.nameservices")) {
+ return true;
+ }
String nameServices = properties.get(PROPERTY_DFS_NAMESERVICES);
if (!StringUtils.isEmpty(nameServices)) {
- String namenodes = properties.get(String.format("dfs.ha.namenodes.%s", nameServices));
- if (!StringUtils.isEmpty(namenodes)) {
- return (namenodes.split(",").length > 1);
+ for (String nameService : nameServices.split(",")) {
+ String namenodes = properties.get(String.format("dfs.ha.namenodes.%s", nameService));
+ if (!StringUtils.isEmpty(namenodes)) {
+ return (namenodes.split(",").length > 1);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/edf8ab24/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
index 3bff060..0bd8f48 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
@@ -880,6 +880,8 @@ public class UpgradeCatalog210Test {
final Map<String, String> propertiesExpectedHdfs = new HashMap<String, String>();
final Map<String, String> propertiesExpectedCoreSite = new HashMap<String, String>();
+ propertiesExpectedHdfs.put("dfs.nameservices", "nncl1,nncl2");
+ propertiesExpectedHdfs.put("dfs.ha.namenodes.nncl2", "nn1,nn2");
propertiesExpectedCoreSite.put("fs.defaultFS", "hdfs://EXAMPLE.COM:8020");
final Injector mockInjector = Guice.createInjector(new AbstractModule() {
@Override
[18/50] [abbrv] ambari git commit: AMBARI-19036. Add default roles to
auto created views. (Ashwin Rajeev via dipayanb)
Posted by nc...@apache.org.
AMBARI-19036. Add default roles to auto created views. (Ashwin Rajeev via dipayanb)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8fbc2718
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8fbc2718
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8fbc2718
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8fbc2718e65a3771a6af9acc48f1b06d13c32bd4
Parents: b2d2ef1
Author: Dipayan Bhowmick <di...@gmail.com>
Authored: Tue Dec 6 01:01:43 2016 +0530
Committer: Dipayan Bhowmick <di...@gmail.com>
Committed: Tue Dec 6 01:03:08 2016 +0530
----------------------------------------------------------------------
.../src/main/resources/view.xml | 23 +++++++++++---------
contrib/views/files/src/main/resources/view.xml | 7 ++++++
.../views/hive-next/src/main/resources/view.xml | 7 ++++++
contrib/views/tez/src/main/resources/view.xml | 7 ++++++
4 files changed, 34 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8fbc2718/contrib/views/capacity-scheduler/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/capacity-scheduler/src/main/resources/view.xml b/contrib/views/capacity-scheduler/src/main/resources/view.xml
index 84a9ee0..cba243c 100644
--- a/contrib/views/capacity-scheduler/src/main/resources/view.xml
+++ b/contrib/views/capacity-scheduler/src/main/resources/view.xml
@@ -23,19 +23,22 @@
<min-ambari-version>2.1.*</min-ambari-version>
<cluster-config-options>AMBARI-ONLY</cluster-config-options>
- <resource>
- <name>scheduler</name>
- <service-class>org.apache.ambari.view.capacityscheduler.CapacitySchedulerService</service-class>
+ <resource>
+ <name>scheduler</name>
+ <service-class>org.apache.ambari.view.capacityscheduler.CapacitySchedulerService</service-class>
</resource>
<auto-instance>
- <name>AUTO_CS_INSTANCE</name>
- <label>YARN Queue Manager</label>
- <description>Manage YARN Capacity Scheduler Queues</description>
- <stack-id>HDP-2.*</stack-id>
- <services>
- <service>YARN</service>
- </services>
+ <name>AUTO_CS_INSTANCE</name>
+ <label>YARN Queue Manager</label>
+ <description>Manage YARN Capacity Scheduler Queues</description>
+ <stack-id>HDP-2.*</stack-id>
+ <services>
+ <service>YARN</service>
+ </services>
+ <roles>
+ <role>CLUSTER.ADMINISTRATOR</role>
+ </roles>
</auto-instance>
</view>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8fbc2718/contrib/views/files/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/view.xml b/contrib/views/files/src/main/resources/view.xml
index d6b2fde..640cee9 100644
--- a/contrib/views/files/src/main/resources/view.xml
+++ b/contrib/views/files/src/main/resources/view.xml
@@ -155,5 +155,12 @@
<services>
<service>HDFS</service>
</services>
+ <roles>
+ <role>CLUSTER.ADMINISTRATOR</role>
+ <role>CLUSTER.OPERATOR</role>
+ <role>SERVICE.ADMINISTRATOR</role>
+ <role>SERVICE.OPERATOR</role>
+ <role>CLUSTER.USER</role>
+ </roles>
</auto-instance>
</view>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8fbc2718/contrib/views/hive-next/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/hive-next/src/main/resources/view.xml b/contrib/views/hive-next/src/main/resources/view.xml
index 02e632c..1107cd1 100644
--- a/contrib/views/hive-next/src/main/resources/view.xml
+++ b/contrib/views/hive-next/src/main/resources/view.xml
@@ -337,5 +337,12 @@
<services>
<service>HIVE</service>
</services>
+ <roles>
+ <role>CLUSTER.ADMINISTRATOR</role>
+ <role>CLUSTER.OPERATOR</role>
+ <role>SERVICE.ADMINISTRATOR</role>
+ <role>SERVICE.OPERATOR</role>
+ <role>CLUSTER.USER</role>
+ </roles>
</auto-instance>
</view>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8fbc2718/contrib/views/tez/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/tez/src/main/resources/view.xml b/contrib/views/tez/src/main/resources/view.xml
index 2bb2bb0..41ffdbc 100644
--- a/contrib/views/tez/src/main/resources/view.xml
+++ b/contrib/views/tez/src/main/resources/view.xml
@@ -91,5 +91,12 @@ limitations under the License. Kerberos, LDAP, Custom. Binary/Htt
<services>
<service>TEZ</service>
</services>
+ <roles>
+ <role>CLUSTER.ADMINISTRATOR</role>
+ <role>CLUSTER.OPERATOR</role>
+ <role>SERVICE.ADMINISTRATOR</role>
+ <role>SERVICE.OPERATOR</role>
+ <role>CLUSTER.USER</role>
+ </roles>
</auto-instance>
</view>
[02/50] [abbrv] ambari git commit: Revert "Revert "AMBARI-18888:
Ambari-agent: Create configuration files with JCEKS information""
Posted by nc...@apache.org.
Revert "Revert "AMBARI-18888: Ambari-agent: Create configuration files with JCEKS information""
Set the value of credential store enabled on the execution command in AmbariManagementController::createHostAction() where the cluster and service are available.
This reverts commit 60a6bd4575fb87fc26c4a277cbabf850ef2089e1.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db46822f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db46822f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db46822f
Branch: refs/heads/branch-dev-patch-upgrade
Commit: db46822fbf50c5e5222bdec094b3a011584f8ee8
Parents: e465c0e
Author: Nahappan Somasundaram <ns...@hortonworks.com>
Authored: Fri Dec 2 08:23:43 2016 -0800
Committer: Nahappan Somasundaram <ns...@hortonworks.com>
Committed: Fri Dec 2 15:39:25 2016 -0800
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 3 +
.../ambari_agent/CustomServiceOrchestrator.py | 120 +++++++++++++++++++
ambari-agent/src/packages/tarball/all.xml | 30 +++++
.../ambari/server/agent/ExecutionCommand.java | 28 +++++
.../ambari/server/agent/HeartBeatHandler.java | 2 +-
.../AmbariManagementControllerImpl.java | 5 +
.../server/agent/TestHeartbeatHandler.java | 28 +++--
7 files changed, 207 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index 61948d4..d6fcf5f 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -46,6 +46,9 @@ keysdir=/var/lib/ambari-agent/keys
server_crt=ca.crt
passphrase_env_var_name=AMBARI_PASSPHRASE
ssl_verify_cert=0
+credential_lib_dir=/var/lib/ambari-agent/cred/lib
+credential_conf_dir=/var/lib/ambari-agent/cred/conf
+credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell
[services]
pidLookupPath=/var/run/
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 11c8cbe..f9ed4cf 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,6 +29,8 @@ from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
from PythonReflectiveExecutor import PythonReflectiveExecutor
+from resource_management.core.utils import PasswordString
+import subprocess
import Constants
import hostname
@@ -65,6 +67,11 @@ class CustomServiceOrchestrator():
REFLECTIVELY_RUN_COMMANDS = FREQUENT_COMMANDS # -- commands which run a lot and often (this increases their speed)
DONT_BACKUP_LOGS_FOR_COMMANDS = FREQUENT_COMMANDS
+ # Path where hadoop credential JARS will be available
+ DEFAULT_CREDENTIAL_SHELL_LIB_PATH = '/var/lib/ambari-agent/cred/lib'
+ DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
+ DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell'
+
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
@@ -78,6 +85,14 @@ class CustomServiceOrchestrator():
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
+ # Construct the hadoop credential lib JARs path
+ self.credential_shell_lib_path = os.path.join(config.get('security', 'credential_lib_dir',
+ self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*')
+
+ self.credential_conf_dir = config.get('security', 'credential_conf_dir', self.DEFAULT_CREDENTIAL_CONF_DIR)
+
+ self.credential_shell_cmd = config.get('security', 'credential_shell_cmd', self.DEFAULT_CREDENTIAL_SHELL_CMD)
+
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -114,6 +129,102 @@ class CustomServiceOrchestrator():
else:
return PythonExecutor(self.tmp_dir, self.config)
+ def getProviderDirectory(self, service_name):
+ """
+ Gets the path to the service conf folder where the JCEKS file will be created.
+
+ :param service_name: Name of the service, for example, HIVE
+ :return: lower case path to the service conf folder
+ """
+
+ # The stack definition scripts of the service can move the
+ # JCEKS file around to where it wants, which is usually
+ # /etc/<service_name>/conf
+
+ conf_dir = os.path.join(self.credential_conf_dir, service_name.lower())
+ if not os.path.exists(conf_dir):
+ os.makedirs(conf_dir, 0644)
+
+ return conf_dir
+
+ def getAffectedConfigTypes(self, commandJson):
+ """
+ Gets the affected config types for the service in this command
+
+ :param commandJson:
+ :return:
+ """
+ return commandJson.get('configuration_attributes')
+
+ def getCredentialProviderPropertyName(self):
+ """
+ Gets the property name used by the hadoop credential provider
+ :return:
+ """
+ return 'hadoop.security.credential.provider.path'
+
+ def generateJceks(self, commandJson):
+ """
+ Generates the JCEKS file with passwords for the service specified in commandJson
+
+ :param commandJson: command JSON
+ :return: An exit value from the external process that generated the JCEKS file. None if
+ there are no passwords in the JSON.
+ """
+ cmd_result = None
+ roleCommand = None
+ if 'roleCommand' in commandJson:
+ roleCommand = commandJson['roleCommand']
+
+ logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
+
+ # Password properties for a config type, if present,
+ # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2}
+ passwordProperties = {}
+ config_types = self.getAffectedConfigTypes(commandJson)
+ for config_type in config_types:
+ elem = config_types.get(config_type)
+ hidden = elem.get('hidden')
+ if hidden is not None:
+ passwordProperties[config_type] = hidden
+
+ # Set up the variables for the external command to generate a JCEKS file
+ java_home = commandJson['hostLevelParams']['java_home']
+ java_bin = '{java_home}/bin/java'.format(java_home=java_home)
+
+ cs_lib_path = self.credential_shell_lib_path
+ serviceName = commandJson['serviceName']
+
+ # Gather the password values and remove them from the configuration
+ configs = commandJson.get('configurations')
+ for key, value in passwordProperties.items():
+ config = configs.get(key)
+ if config is not None:
+ file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key))
+ if os.path.exists(file_path):
+ os.remove(file_path)
+ provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
+ logger.info('provider_path={0}'.format(provider_path))
+ for alias in value:
+ pwd = config.get(alias)
+ if pwd is not None:
+ # Remove the clear text password
+ config.pop(alias, None)
+ # Add JCEKS provider path instead
+ config[self.getCredentialProviderPropertyName()] = provider_path
+ logger.debug("config={0}".format(config))
+ protected_pwd = PasswordString(pwd)
+ # Generate the JCEKS file
+ cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
+ alias, '-value', protected_pwd, '-provider', provider_path)
+ logger.info(cmd)
+ cmd_result = subprocess.call(cmd)
+ logger.info('cmd_result = {0}'.format(cmd_result))
+ os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
+
+ return cmd_result
+
+
def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name=None,
override_output_files=True, retry=False):
"""
@@ -174,6 +285,15 @@ class CustomServiceOrchestrator():
handle.on_background_command_started = self.map_task_to_process
del command['__handle']
+ # If command contains credentialStoreEnabled, then
+ # generate the JCEKS file for the configurations.
+ credentialStoreEnabled = False
+ if 'credentialStoreEnabled' in command:
+ credentialStoreEnabled = (command['credentialStoreEnabled'] == "true")
+
+ if credentialStoreEnabled == True:
+ self.generateJceks(command)
+
json_path = self.dump_command_to_json(command, retry)
pre_hook_tuple = self.resolve_hook_script_path(hook_dir,
self.PRE_HOOK_PREFIX, command_name, script_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-agent/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/src/packages/tarball/all.xml b/ambari-agent/src/packages/tarball/all.xml
index c481208..f8a54e3 100644
--- a/ambari-agent/src/packages/tarball/all.xml
+++ b/ambari-agent/src/packages/tarball/all.xml
@@ -190,4 +190,34 @@
<outputDirectory>/var/lib/${project.artifactId}/data</outputDirectory>
</file>
</files>
+ <moduleSets>
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
+ <unpack>false</unpack>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
+ <unpack>false</unpack>
+ <includes>
+ <include>commons-cli:commons-cli</include>
+ <include>commons-collections:commons-collections</include>
+ <include>commons-configuration:commons-configuration</include>
+ <include>commons-io:commons-io:jar:${commons.io.version}</include>
+ <include>commons-lang:commons-lang</include>
+ <include>commons-logging:commons-logging</include>
+ <include>com.google.guava:guava</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.apache.hadoop:hadoop-common</include>
+ <include>org.apache.hadoop:hadoop-auth</include>
+ <include>org.apache.htrace:htrace-core</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ </moduleSet>
+ </moduleSets>
</assembly>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index ef1ee4f..e46167a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -112,6 +112,13 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("availableServices")
private Map<String, String> availableServices = new HashMap<>();
+ /**
+ * "true" or "false" indicating whether this
+ * service is enabled for credential store use.
+ */
+ @SerializedName("credentialStoreEnabled")
+ private String credentialStoreEnabled;
+
public String getCommandId() {
return commandId;
}
@@ -295,6 +302,27 @@ public class ExecutionCommand extends AgentCommand {
this.serviceType = serviceType;
}
+ /**
+ * Get a value indicating whether this service is enabled
+ * for credential store use.
+ *
+ * @return "true" or "false", any other value is
+ * considered as "false"
+ */
+ public String getCredentialStoreEnabled() {
+ return credentialStoreEnabled;
+ }
+
+ /**
+ * Set a value indicating whether this service is enabled
+ * for credential store use.
+ *
+ * @param credentialStoreEnabled
+ */
+ public void setCredentialStoreEnabled(String credentialStoreEnabled) {
+ this.credentialStoreEnabled = credentialStoreEnabled;
+ }
+
public String getComponentName() {
return componentName;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
index 75bef30..a25b875 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
@@ -329,7 +329,7 @@ public class HeartBeatHandler {
case BACKGROUND_EXECUTION_COMMAND:
case EXECUTION_COMMAND: {
ExecutionCommand ec = (ExecutionCommand)ac;
- LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, taskId {}",
+ LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, task ID {}",
ec.getHostname(), ec.getRole(), ec.getRoleCommand(), ec.getCommandId(), ec.getTaskId());
Map<String, String> hlp = ec.getHostLevelParams();
if (hlp != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 0448b9f..aa35f09 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2118,6 +2118,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
+ // Get the value of credential store enabled from the DB
+ Service clusterService = cluster.getService(serviceName);
+ execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));
+
// Create a local copy for each command
Map<String, String> commandParams = new TreeMap<String, String>();
if (commandParamsInp != null) { // if not defined
@@ -2324,6 +2328,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()));
+
if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) {
LOG.debug("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}",
execCmd.getHostname(), execCmd.getRole(), execCmd.getRoleCommand(), execCmd.getCommandId(), execCmd.getConfigurationTags().get("cluster-env").get("tag"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/db46822f/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index a50a116..ac58f64 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -48,6 +48,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -171,22 +172,31 @@ public class TestHeartbeatHandler {
ActionManager am = actionManagerTestHelper.getMockActionManager();
expect(am.getTasks(anyObject(List.class))).andReturn(new ArrayList<HostRoleCommand>());
replay(am);
+
+ Cluster cluster = heartbeatTestHelper.getDummyCluster();
+ Service hdfs = cluster.addService(HDFS);
+ hdfs.addServiceComponent(DATANODE);
+ hdfs.addServiceComponent(NAMENODE);
+ hdfs.addServiceComponent(SECONDARY_NAMENODE);
+ Collection<Host> hosts = cluster.getHosts();
+ assertEquals(hosts.size(), 1);
+
Clusters fsm = clusters;
- fsm.addHost(DummyHostname1);
- Host hostObject = clusters.getHost(DummyHostname1);
+ Host hostObject = hosts.iterator().next();
hostObject.setIPv4("ipv4");
hostObject.setIPv6("ipv6");
hostObject.setOsType(DummyOsType);
+ String hostname = hostObject.getHostName();
ActionQueue aq = new ActionQueue();
HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector);
Register reg = new Register();
HostInfo hi = new HostInfo();
- hi.setHostName(DummyHostname1);
+ hi.setHostName(hostname);
hi.setOS(DummyOs);
hi.setOSRelease(DummyOSRelease);
- reg.setHostname(DummyHostname1);
+ reg.setHostname(hostname);
reg.setHardwareProfile(hi);
reg.setAgentVersion(metaInfo.getServerVersion());
handler.handleRegistration(reg);
@@ -195,19 +205,21 @@ public class TestHeartbeatHandler {
ExecutionCommand execCmd = new ExecutionCommand();
execCmd.setRequestAndStage(2, 34);
- execCmd.setHostname(DummyHostname1);
- aq.enqueue(DummyHostname1, new ExecutionCommand());
+ execCmd.setHostname(hostname);
+ execCmd.setClusterName(cluster.getClusterName());
+ execCmd.setServiceName(HDFS);
+ aq.enqueue(hostname, execCmd);
HeartBeat hb = new HeartBeat();
hb.setResponseId(0);
HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus);
List<Alert> al = new ArrayList<Alert>();
al.add(new Alert());
hb.setNodeStatus(hs);
- hb.setHostname(DummyHostname1);
+ hb.setHostname(hostname);
handler.handleHeartBeat(hb);
assertEquals(HostState.HEALTHY, hostObject.getState());
- assertEquals(0, aq.dequeueAll(DummyHostname1).size());
+ assertEquals(0, aq.dequeueAll(hostname).size());
}
[27/50] [abbrv] ambari git commit: AMBARI-19085 Host Ordered upgrade
should not show as an Upgrade option in UI. (atkach)
Posted by nc...@apache.org.
AMBARI-19085 Host Ordered upgrade should not show as an Upgrade option in UI. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a2f7a030
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a2f7a030
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a2f7a030
Branch: refs/heads/branch-dev-patch-upgrade
Commit: a2f7a03088f553c51be40c16c1b56948a19a15b6
Parents: 14c3969
Author: Andrii Tkach <at...@apache.org>
Authored: Tue Dec 6 12:20:50 2016 +0200
Committer: Andrii Tkach <at...@apache.org>
Committed: Tue Dec 6 12:45:18 2016 +0200
----------------------------------------------------------------------
.../main/admin/stack_and_upgrade_controller.js | 13 ++++-
.../admin/stack_upgrade/upgrade_options.hbs | 58 ++++++++++----------
.../admin/stack_upgrade/upgrade_wizard_view.js | 6 +-
.../stack_upgrade/upgrade_wizard_view_test.js | 2 +-
4 files changed, 48 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2f7a030/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js b/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
index 35756cd..6d97205 100644
--- a/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
+++ b/ambari-web/app/controllers/main/admin/stack_and_upgrade_controller.js
@@ -52,6 +52,16 @@ App.MainAdminStackAndUpgradeController = Em.Controller.extend(App.LocalStorage,
upgradeType: null,
/**
+ * @type {Em.Object}
+ */
+ upgradeTypeConfig: Em.computed.findByKey('upgradeMethods', 'type', 'upgradeType'),
+
+ /**
+ * @type {boolean}
+ */
+ cantBeStarted: Em.computed.alias('upgradeTypeConfig.cantBeStarted'),
+
+ /**
* @type {boolean}
* @default true
*/
@@ -171,7 +181,8 @@ App.MainAdminStackAndUpgradeController = Em.Controller.extend(App.LocalStorage,
precheckResultsMessage: '',
precheckResultsTitle: '',
action: '',
- isWizardRestricted: !App.supports.enabledWizardForHostOrderedUpgrade
+ isWizardRestricted: !App.supports.enabledWizardForHostOrderedUpgrade,
+ cantBeStarted: true
})
],
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2f7a030/ambari-web/app/templates/main/admin/stack_upgrade/upgrade_options.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/admin/stack_upgrade/upgrade_options.hbs b/ambari-web/app/templates/main/admin/stack_upgrade/upgrade_options.hbs
index 18fcfcf..fe3833c 100644
--- a/ambari-web/app/templates/main/admin/stack_upgrade/upgrade_options.hbs
+++ b/ambari-web/app/templates/main/admin/stack_upgrade/upgrade_options.hbs
@@ -26,35 +26,37 @@
<div class="text method-text">{{t admin.stackVersions.version.upgrade.upgradeOptions.bodyMsg.method}}</div>
<div {{bindAttr class=":row :method-options view.isInUpgradeWizard:disabled"}}>
{{#each method in view.upgradeMethods}}
- <div class="method-option col-md-6">
- <div {{bindAttr class="method.allowed::not-allowed method.allowed::not-allowed-by-version
- method.isPrecheckFailed:not-allowed method.isPrecheckFailed:check-failed
- method.selected:selected method.type :img-thumbnail"}}
- {{action selectMethod method target="view"}}>
- <div {{bindAttr class="method.icon :method-icon"}}></div>
- <div class="method-name">{{method.displayName}}</div>
- <div class="method-description">{{{method.description}}}</div>
- {{#if view.showPreUpgradeChecks}}
- {{#if method.isCheckRequestInProgress}}
- <div class="method-precheck-message checking">
- {{view App.SpinnerView message="admin.stackVersions.version.upgrade.upgradeOptions.preCheck.msg.checking"}}
- </div>
- {{else}}
- <div {{bindAttr class=":method-precheck-message method.precheckResultsMessageClass"}}>
- <i {{bindAttr class="method.precheckResultsMessageIconClass"}}></i>
- <b>{{method.precheckResultsTitle}}</b>
- <a {{action runAction method target="view"}}>
- {{method.precheckResultsMessage}}
- </a>
- {{#if method.bypassedFailures}}
- <div
- class="alert-danger">{{t admin.stackVersions.version.upgrade.upgradeOptions.errors_bypassed}}</div>
- {{/if}}
- </div>
+ {{#unless method.cantBeStarted}}
+ <div class="method-option col-md-6">
+ <div {{bindAttr class="method.allowed::not-allowed method.allowed::not-allowed-by-version
+ method.isPrecheckFailed:not-allowed method.isPrecheckFailed:check-failed
+ method.selected:selected method.type :img-thumbnail"}}
+ {{action selectMethod method target="view"}}>
+ <div {{bindAttr class="method.icon :method-icon"}}></div>
+ <div class="method-name">{{method.displayName}}</div>
+ <div class="method-description">{{{method.description}}}</div>
+ {{#if view.showPreUpgradeChecks}}
+ {{#if method.isCheckRequestInProgress}}
+ <div class="method-precheck-message checking">
+ {{view App.SpinnerView message="admin.stackVersions.version.upgrade.upgradeOptions.preCheck.msg.checking"}}
+ </div>
+ {{else}}
+ <div {{bindAttr class=":method-precheck-message method.precheckResultsMessageClass"}}>
+ <i {{bindAttr class="method.precheckResultsMessageIconClass"}}></i>
+ <b>{{method.precheckResultsTitle}}</b>
+ <a {{action runAction method target="view"}}>
+ {{method.precheckResultsMessage}}
+ </a>
+ {{#if method.bypassedFailures}}
+ <div
+ class="alert-danger">{{t admin.stackVersions.version.upgrade.upgradeOptions.errors_bypassed}}</div>
+ {{/if}}
+ </div>
+ {{/if}}
{{/if}}
- {{/if}}
- </div>
- </div>
+ </div>
+ </div>
+ {{/unless}}
{{/each}}
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2f7a030/ambari-web/app/views/main/admin/stack_upgrade/upgrade_wizard_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/admin/stack_upgrade/upgrade_wizard_view.js b/ambari-web/app/views/main/admin/stack_upgrade/upgrade_wizard_view.js
index e54a0fa..bcb1ad7 100644
--- a/ambari-web/app/views/main/admin/stack_upgrade/upgrade_wizard_view.js
+++ b/ambari-web/app/views/main/admin/stack_upgrade/upgrade_wizard_view.js
@@ -68,7 +68,11 @@ App.upgradeWizardView = Em.View.extend({
* when downgrade already started
* @type {boolean}
*/
- isDowngradeAvailable: Em.computed.and('!controller.isDowngrade', 'controller.downgradeAllowed'),
+ isDowngradeAvailable: Em.computed.and(
+ '!controller.isDowngrade',
+ 'controller.downgradeAllowed',
+ '!controller.cantBeStarted'
+ ),
/**
* progress value is rounded to floor
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2f7a030/ambari-web/test/views/main/admin/stack_upgrade/upgrade_wizard_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/main/admin/stack_upgrade/upgrade_wizard_view_test.js b/ambari-web/test/views/main/admin/stack_upgrade/upgrade_wizard_view_test.js
index 5630a8c..075bcc8 100644
--- a/ambari-web/test/views/main/admin/stack_upgrade/upgrade_wizard_view_test.js
+++ b/ambari-web/test/views/main/admin/stack_upgrade/upgrade_wizard_view_test.js
@@ -416,7 +416,7 @@ describe('App.upgradeWizardView', function () {
});
});
- App.TestAliases.testAsComputedAnd(getView(), 'isDowngradeAvailable', ['!controller.isDowngrade', 'controller.downgradeAllowed']);
+ App.TestAliases.testAsComputedAnd(getView(), 'isDowngradeAvailable', ['!controller.isDowngrade', 'controller.downgradeAllowed', '!controller.cantBeStarted']);
describe("#taskDetails", function () {
it("runningItem present", function () {
[49/50] [abbrv] ambari git commit: AMBARI-19125. Active tab is not
switched (onechiporenko)
Posted by nc...@apache.org.
AMBARI-19125. Active tab is not switched (onechiporenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/234838f4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/234838f4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/234838f4
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 234838f4b6651dbebf09d57910aea925c8247f1c
Parents: 69cdaa7
Author: Oleg Nechiporenko <on...@apache.org>
Authored: Wed Dec 7 15:01:08 2016 +0200
Committer: Oleg Nechiporenko <on...@apache.org>
Committed: Wed Dec 7 15:01:08 2016 +0200
----------------------------------------------------------------------
ambari-web/app/templates/common/host_progress_popup.hbs | 8 +++++---
ambari-web/app/views/main/host/menu.js | 4 ++--
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/234838f4/ambari-web/app/templates/common/host_progress_popup.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/common/host_progress_popup.hbs b/ambari-web/app/templates/common/host_progress_popup.hbs
index f0a8c17..ae411b1 100644
--- a/ambari-web/app/templates/common/host_progress_popup.hbs
+++ b/ambari-web/app/templates/common/host_progress_popup.hbs
@@ -266,9 +266,11 @@
<div class="col-sm-12">
{{#if App.supports.logSearch}}
{{#if view.isLogSearchInstalled}}
- <button type="button" class="btn btn-link pull-right" {{action navigateToHostLogs target="view"}} {{bindAttr class="view.isLogsLinkVisible::hidden"}}>
- <i class="glyphicon glyphicon-file"></i> {{t common.host}} {{t common.logs}}
- </button>
+ {{#isAuthorized "SERVICE.VIEW_OPERATIONAL_LOGS"}}
+ <button type="button" class="btn btn-link pull-right" {{action navigateToHostLogs target="view"}} {{bindAttr class="view.isLogsLinkVisible::hidden"}}>
+ <i class="glyphicon glyphicon-file"></i> {{t common.host}} {{t common.logs}}
+ </button>
+ {{/isAuthorized}}
{{/if}}
{{/if}}
<button type="button" class="btn btn-link pull-right" {{translateAttr title="common.openNewWindow"}} {{action openTaskLogInDialog}}>
http://git-wip-us.apache.org/repos/asf/ambari/blob/234838f4/ambari-web/app/views/main/host/menu.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/host/menu.js b/ambari-web/app/views/main/host/menu.js
index b3c52f4..4ebc140 100644
--- a/ambari-web/app/views/main/host/menu.js
+++ b/ambari-web/app/views/main/host/menu.js
@@ -95,9 +95,9 @@ App.MainHostMenuView = Em.CollectionView.extend({
activateView: function () {
var defaultRoute = App.router.get('currentState.name') || "summary";
$.each(this._childViews, function () {
- this.set('active', (this.get('content.routing') == defaultRoute ? "active" : ""));
+ this.set('active', this.get('content.routing') === defaultRoute ? 'active' : '');
});
- },
+ }.observes('App.router.currentState.name'),
deactivateChildViews: function () {
this.get('_childViews').setEach('active', '');
[25/50] [abbrv] ambari git commit: AMBARI-18981 - Manage JournalNodes
Wizard: Display only the steps needed for DELETE ONLY task (rzang)
Posted by nc...@apache.org.
AMBARI-18981 - Manage JournalNodes Wizard: Display only the steps needed for DELETE ONLY task (rzang)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/353b2ac0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/353b2ac0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/353b2ac0
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 353b2ac0064e0d014ace50acba12c523491782a8
Parents: e88f846
Author: Richard Zang <rz...@apache.org>
Authored: Mon Dec 5 18:11:12 2016 -0800
Committer: Richard Zang <rz...@apache.org>
Committed: Mon Dec 5 18:11:12 2016 -0800
----------------------------------------------------------------------
.../app/controllers/global/update_controller.js | 5 ++--
.../journalNode/wizard_controller.js | 28 +++++++++++++-------
.../app/routes/manage_journalnode_routes.js | 4 +--
.../highAvailability/journalNode/wizard.hbs | 13 +++++----
.../highAvailability/journalNode/step2_view.js | 5 ++--
5 files changed, 30 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/353b2ac0/ambari-web/app/controllers/global/update_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/global/update_controller.js b/ambari-web/app/controllers/global/update_controller.js
index 4631f09..ce93b34 100644
--- a/ambari-web/app/controllers/global/update_controller.js
+++ b/ambari-web/app/controllers/global/update_controller.js
@@ -249,11 +249,10 @@ App.UpdateController = Em.Controller.extend({
]);
}
else {
- // clusterController.isHostsLoaded may be changed in callback, that is why it's value is cached before calling callback
- isHostsLoaded = App.router.get('clusterController.isHostsLoaded');
- callback();
// On pages except for hosts/hostDetails, making sure hostsMapper loaded only once on page load, no need to update, but at least once
+ isHostsLoaded = App.router.get('clusterController.isHostsLoaded');
if (isHostsLoaded) {
+ callback();
return;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/353b2ac0/ambari-web/app/controllers/main/admin/highAvailability/journalNode/wizard_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/admin/highAvailability/journalNode/wizard_controller.js b/ambari-web/app/controllers/main/admin/highAvailability/journalNode/wizard_controller.js
index 2305524..fe9a15b 100644
--- a/ambari-web/app/controllers/main/admin/highAvailability/journalNode/wizard_controller.js
+++ b/ambari-web/app/controllers/main/admin/highAvailability/journalNode/wizard_controller.js
@@ -120,22 +120,30 @@ App.ManageJournalNodeWizardController = App.WizardController.extend({
},
getJournalNodesToAdd: function () {
- return this.get('content.masterComponentHosts').filterProperty('component', 'JOURNALNODE')
- .filterProperty('isInstalled', false).mapProperty('hostName');
+ var result = [];
+ var masterComponentHosts = this.get('content.masterComponentHosts');
+ if (masterComponentHosts) {
+ result = masterComponentHosts.filterProperty('component', 'JOURNALNODE').filterProperty('isInstalled', false).mapProperty('hostName');
+ }
+ return result;
},
getJournalNodesToDelete: function () {
- var existingHosts = App.HostComponent.find().filterProperty('componentName', 'JOURNALNODE').mapProperty('hostName');
- var currentJNs = this.get('content.masterComponentHosts').filterProperty('component', 'JOURNALNODE');
- var removed = existingHosts.filter(function(host) {
- return currentJNs.filterProperty('hostName', host).length == 0;
- });
- return removed;
+ var result = [];
+ var masterComponentHosts = this.get('content.masterComponentHosts');
+ if (masterComponentHosts) {
+ var currentJNs = masterComponentHosts.filterProperty('component', 'JOURNALNODE');
+ var existingHosts = App.HostComponent.find().filterProperty('componentName', 'JOURNALNODE').mapProperty('hostName');
+ result = existingHosts.filter(function(host) {
+ return currentJNs.filterProperty('hostName', host).length == 0;
+ });
+ }
+ return result;
},
isDeleteOnly: function () {
- return this.getJournalNodesToAdd().length == 0 && this.getJournalNodesToDelete().length > 0;
- },
+ return this.get('currentStep') > 1 && this.getJournalNodesToAdd().length == 0 && this.getJournalNodesToDelete().length > 0;
+ }.property('content.masterComponentHosts', 'App.router.clusterController.isHostsLoaded', 'currentStep'),
/**
* Save config properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/353b2ac0/ambari-web/app/routes/manage_journalnode_routes.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/routes/manage_journalnode_routes.js b/ambari-web/app/routes/manage_journalnode_routes.js
index e13a338..90b45b0 100644
--- a/ambari-web/app/routes/manage_journalnode_routes.js
+++ b/ambari-web/app/routes/manage_journalnode_routes.js
@@ -118,7 +118,7 @@ module.exports = App.WizardRoute.extend({
controller.saveConfigTag(stepController.get("hdfsSiteTag"));
controller.saveNameServiceId(stepController.get('content.nameServiceId'));
App.set('router.nextBtnClickInProgress', false);
- if (controller.isDeleteOnly()) {
+ if (controller.get('isDeleteOnly')) {
router.transitionTo('step4');
} else {
router.transitionTo('step3');
@@ -167,7 +167,7 @@ module.exports = App.WizardRoute.extend({
next: function (router) {
var controller = router.get('manageJournalNodeWizardController');
controller.clearTasksData();
- if (controller.isDeleteOnly()) {
+ if (controller.get('isDeleteOnly')) {
router.transitionTo('step8');
} else {
router.transitionTo('step5');
http://git-wip-us.apache.org/repos/asf/ambari/blob/353b2ac0/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs b/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
index 844e1c8..a4c2155 100644
--- a/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
+++ b/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
@@ -22,13 +22,12 @@
<ul class="nav nav-pills nav-stacked">
<li {{bindAttr class="isStep1:active view.isStep1Disabled:disabled view.isStep1Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep1 target="controller"}}><i class="step-marker"><span class="step-index">1</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step1.header}}</p></a></li>
<li {{bindAttr class="isStep2:active view.isStep2Disabled:disabled view.isStep2Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep2 target="controller"}}><i class="step-marker"><span class="step-index">2</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step2.header}}</p></a></li>
- <li {{bindAttr class="isStep3:active view.isStep3Disabled:disabled view.isStep3Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep3 target="controller"}}><i class="step-marker"><span class="step-index">3</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step3.header}}</p></a></li>
- <li {{bindAttr class="isStep4:active view.isStep4Disabled:disabled view.isStep4Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep4 target="controller"}}><i class="step-marker"><span class="step-index">4</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step4.header}}</p></a></li>
- <li {{bindAttr class="isStep5:active view.isStep5Disabled:disabled view.isStep5Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep5 target="controller"}}><i class="step-marker"><span class="step-index">5</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step5.header}}</p></a></li>
- <li {{bindAttr class="isStep6:active view.isStep6Disabled:disabled view.isStep6Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep6 target="controller"}}><i class="step-marker"><span class="step-index">6</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step6.header}}</p></a></li>
- <li {{bindAttr class="isStep7:active view.isStep7Disabled:disabled view.isStep7Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep5 target="controller"}}><i class="step-marker"><span class="step-index">7</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step7.header}}</p></a></li>
- <li {{bindAttr class="isStep8:active view.isStep8Disabled:disabled view.isStep8Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep6 target="controller"}}><i class="step-marker"><span class="step-index">8</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step8.header}}</p></a></li>
-
+ <li {{bindAttr class="isStep3:active view.isStep3Disabled:disabled view.isStep3Completed:completed isDeleteOnly:hidden"}}><a href="javascript:void(null);" {{action gotoStep3 target="controller"}}><i class="step-marker"><span class="step-index">3</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step3.header}}</p></a></li>
+ <li {{bindAttr class="isStep4:active view.isStep4Disabled:disabled view.isStep4Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep4 target="controller"}}><i class="step-marker"><span class="step-index">{{#if isDeleteOnly}}3{{else}}4{{/if}}</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step4.header}}</p></a></li>
+ <li {{bindAttr class="isStep5:active view.isStep5Disabled:disabled view.isStep5Completed:completed isDeleteOnly:hidden"}}><a href="javascript:void(null);" {{action gotoStep5 target="controller"}}><i class="step-marker"><span class="step-index">5</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step5.header}}</p></a></li>
+ <li {{bindAttr class="isStep6:active view.isStep6Disabled:disabled view.isStep6Completed:completed isDeleteOnly:hidden"}}><a href="javascript:void(null);" {{action gotoStep6 target="controller"}}><i class="step-marker"><span class="step-index">6</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step6.header}}</p></a></li>
+ <li {{bindAttr class="isStep7:active view.isStep7Disabled:disabled view.isStep7Completed:completed isDeleteOnly:hidden"}}><a href="javascript:void(null);" {{action gotoStep5 target="controller"}}><i class="step-marker"><span class="step-index">7</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step7.header}}</p></a></li>
+ <li {{bindAttr class="isStep8:active view.isStep8Disabled:disabled view.isStep8Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep6 target="controller"}}><i class="step-marker"><span class="step-index">{{#if isDeleteOnly}}4{{else}}8{{/if}}</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step8.header}}</p></a></li>
</ul>
</div>
{{outlet}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/353b2ac0/ambari-web/app/views/main/admin/highAvailability/journalNode/step2_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/admin/highAvailability/journalNode/step2_view.js b/ambari-web/app/views/main/admin/highAvailability/journalNode/step2_view.js
index 656c487..680daa5 100644
--- a/ambari-web/app/views/main/admin/highAvailability/journalNode/step2_view.js
+++ b/ambari-web/app/views/main/admin/highAvailability/journalNode/step2_view.js
@@ -22,18 +22,17 @@ var App = require('app');
App.ManageJournalNodeWizardStep2View = Em.View.extend({
templateName: require('templates/main/admin/highAvailability/journalNode/step2'),
- aaa: '',
didInsertElement: function () {
this.get('controller').loadStep();
},
journalNodesToAdd: function () {
return App.router.get('manageJournalNodeWizardController').getJournalNodesToAdd();
- }.property(),
+ }.property('App.router.clusterController.isHostsLoaded'),
journalNodesToDelete: function () {
return App.router.get('manageJournalNodeWizardController').getJournalNodesToDelete();
- }.property(),
+ }.property('App.router.clusterController.isHostsLoaded'),
isBackButtonVisible: false
});
[35/50] [abbrv] ambari git commit: AMBARI-19091. HDP 3.0 TP -
bootstrap the stack with metainfo, cluster-env, repos, and hooks (alejandro)
Posted by nc...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
new file mode 100644
index 0000000..d838211
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/params.py
@@ -0,0 +1,326 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import default
+from resource_management.libraries.functions import format_jvm_option
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import format_stack_version, compare_versions
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+
+config = Script.get_config()
+
+# Whether to skip copying fast-hdfs-resource.jar to /var/lib/ambari-agent/lib/
+# This is required if tarballs are going to be copied to HDFS, so set to False
+sysprep_skip_copy_fast_jar_hdfs = default("/configurations/cluster-env/sysprep_skip_copy_fast_jar_hdfs", False)
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+dfs_type = default("/commandParams/dfs_type", "")
+stack_root = Script.get_stack_root()
+hadoop_conf_dir = "/etc/hadoop/conf"
+component_list = default("/localComponents", [])
+
+hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
+
+hadoop_metrics2_properties_content = config['configurations']['hadoop-metrics2.properties']['content']
+
+# hadoop default params
+mapreduce_libs_path = format("{stack_root}/current/hadoop-mapreduce-client/*")
+
+hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
+hadoop_lib_home = stack_select.get_hadoop_dir("lib")
+hadoop_bin = stack_select.get_hadoop_dir("sbin")
+hadoop_home = stack_select.get_hadoop_dir("home")
+create_lib_snappy_symlinks = False
+
+
+current_service = config['serviceName']
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#users and groups
+has_hadoop_env = 'hadoop-env' in config['configurations']
+mapred_user = config['configurations']['mapred-env']['mapred_user']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+yarn_user = config['configurations']['yarn-env']['yarn_user']
+
+user_group = config['configurations']['cluster-env']['user_group']
+
+#hosts
+hostname = config["hostname"]
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+rm_host = default("/clusterHostInfo/rm_host", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
+hive_server_host = default("/clusterHostInfo/hive_server_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+hs_host = default("/clusterHostInfo/hs_host", [])
+jtnode_host = default("/clusterHostInfo/jtnode_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+
+has_namenode = not len(namenode_host) == 0
+has_resourcemanager = not len(rm_host) == 0
+has_slaves = not len(slave_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_hcat_server_host = not len(hcat_server_hosts) == 0
+has_hive_server_host = not len(hive_server_host) == 0
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_zk_host = not len(zk_hosts) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_metric_collector = not len(ams_collector_hosts) == 0
+
+is_namenode_master = hostname in namenode_host
+is_jtnode_master = hostname in jtnode_host
+is_rmnode_master = hostname in rm_host
+is_hsnode_master = hostname in hs_host
+is_hbase_master = hostname in hbase_master_hosts
+is_slave = hostname in slave_hosts
+
+if has_ganglia_server:
+ ganglia_server_host = ganglia_server_hosts[0]
+
+metric_collector_port = None
+if has_metric_collector:
+ if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ else:
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
+ if metric_collector_web_address.find(':') != -1:
+ metric_collector_port = metric_collector_web_address.split(':')[1]
+ else:
+ metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+ metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
+
+ pass
+metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
+metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
+
+# Cluster Zookeeper quorum
+zookeeper_quorum = None
+if has_zk_host:
+ if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+ else:
+ zookeeper_clientPort = '2181'
+ zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+ # last port config
+ zookeeper_quorum += ':' + zookeeper_clientPort
+
+#hadoop params
+
+if has_namenode or dfs_type == 'HCFS':
+ hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}")
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+ task_log4j_properties_location = os.path.join(hadoop_conf_dir, "task-log4j.properties")
+
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hbase_tmp_dir = "/tmp/hbase-hbase"
+#db params
+server_db_name = config['hostLevelParams']['db_name']
+db_driver_filename = config['hostLevelParams']['db_driver_filename']
+oracle_driver_url = config['hostLevelParams']['oracle_jdbc_url']
+mysql_driver_url = config['hostLevelParams']['mysql_jdbc_url']
+ambari_server_resources = config['hostLevelParams']['jdk_location']
+oracle_driver_symlink_url = format("{ambari_server_resources}oracle-jdbc-driver.jar")
+mysql_driver_symlink_url = format("{ambari_server_resources}mysql-jdbc-driver.jar")
+
+ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url'][0]
+ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'][0]
+ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'][0]
+ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'][0]
+
+if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']:
+ rca_enabled = config['configurations']['hadoop-env']['rca_enabled']
+else:
+ rca_enabled = False
+rca_disabled_prefix = "###"
+if rca_enabled == True:
+ rca_prefix = ""
+else:
+ rca_prefix = rca_disabled_prefix
+
+#hadoop-env.sh
+java_home = config['hostLevelParams']['java_home']
+
+jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+#log4j.properties
+
+yarn_log_dir_prefix = default("/configurations/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn")
+
+dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
+
+#log4j.properties
+if (('hdfs-log4j' in config['configurations']) and ('content' in config['configurations']['hdfs-log4j'])):
+ log4j_props = config['configurations']['hdfs-log4j']['content']
+ if (('yarn-log4j' in config['configurations']) and ('content' in config['configurations']['yarn-log4j'])):
+ log4j_props += config['configurations']['yarn-log4j']['content']
+else:
+ log4j_props = None
+
+refresh_topology = False
+command_params = config["commandParams"] if "commandParams" in config else None
+if command_params is not None:
+ refresh_topology = bool(command_params["refresh_topology"]) if "refresh_topology" in command_params else False
+
+ambari_libs_dir = "/var/lib/ambari-agent/lib"
+is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+#host info
+all_hosts = default("/clusterHostInfo/all_hosts", [])
+all_racks = default("/clusterHostInfo/all_racks", [])
+all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+
+#topology files
+net_topology_script_file_path = "/etc/hadoop/conf/topology_script.py"
+net_topology_script_dir = os.path.dirname(net_topology_script_file_path)
+net_topology_mapping_data_file_name = 'topology_mappings.data'
+net_topology_mapping_data_file_path = os.path.join(net_topology_script_dir, net_topology_mapping_data_file_name)
+
+#Added logic to create /tmp and /user directory for HCFS stack.
+has_core_site = 'core-site' in config['configurations']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+kinit_path_local = get_kinit_path()
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None)
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+smoke_hdfs_user_dir = format("/user/{smoke_user}")
+smoke_hdfs_user_mode = 0770
+
+
+##### Namenode RPC ports - metrics config section start #####
+
+# Figure out the rpc ports for current namenode
+nn_rpc_client_port = None
+nn_rpc_dn_port = None
+nn_rpc_healthcheck_port = None
+
+namenode_id = None
+namenode_rpc = None
+
+dfs_ha_enabled = False
+dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None)
+if dfs_ha_nameservices is None:
+ dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None)
+dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None)
+
+dfs_ha_namemodes_ids_list = []
+other_namenode_id = None
+
+if dfs_ha_namenode_ids:
+ dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",")
+ dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list)
+ if dfs_ha_namenode_ids_array_len > 1:
+ dfs_ha_enabled = True
+
+if dfs_ha_enabled:
+ for nn_id in dfs_ha_namemodes_ids_list:
+ nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')]
+ if hostname in nn_host:
+ namenode_id = nn_id
+ namenode_rpc = nn_host
+ pass
+ pass
+else:
+ namenode_rpc = default('/configurations/hdfs-site/dfs.namenode.rpc-address', None)
+
+if namenode_rpc:
+ nn_rpc_client_port = namenode_rpc.split(':')[1].strip()
+
+if dfs_ha_enabled:
+ dfs_service_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.servicerpc-address.{dfs_ha_nameservices}.{namenode_id}'), None)
+ dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address.{dfs_ha_nameservices}.{namenode_id}'), None)
+else:
+ dfs_service_rpc_address = default('/configurations/hdfs-site/dfs.namenode.servicerpc-address', None)
+ dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address'), None)
+
+if dfs_service_rpc_address:
+ nn_rpc_dn_port = dfs_service_rpc_address.split(':')[1].strip()
+
+if dfs_lifeline_rpc_address:
+ nn_rpc_healthcheck_port = dfs_lifeline_rpc_address.split(':')[1].strip()
+
+is_nn_client_port_configured = False if nn_rpc_client_port is None else True
+is_nn_dn_port_configured = False if nn_rpc_dn_port is None else True
+is_nn_healthcheck_port_configured = False if nn_rpc_healthcheck_port is None else True
+
+##### end #####
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/rack_awareness.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/rack_awareness.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/rack_awareness.py
new file mode 100644
index 0000000..548f051
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/rack_awareness.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.resources import File
+from resource_management.core.source import StaticFile, Template
+from resource_management.libraries.functions import format
+
+
+def create_topology_mapping():
+ import params
+
+ File(params.net_topology_mapping_data_file_path,
+ content=Template("topology_mappings.data.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group,
+ only_if=format("test -d {net_topology_script_dir}"))
+
+def create_topology_script():
+ import params
+
+ File(params.net_topology_script_file_path,
+ content=StaticFile('topology_script.py'),
+ mode=0755,
+ only_if=format("test -d {net_topology_script_dir}"))
+
+def create_topology_script_and_mapping():
+ import params
+ if params.has_hadoop_env:
+ create_topology_mapping()
+ create_topology_script()
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py
new file mode 100644
index 0000000..5dce8e0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/shared_initialization.py
@@ -0,0 +1,191 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
+
+from resource_management import *
+
+def setup_hadoop():
+ """
+ Setup hadoop files and directories
+ """
+ import params
+
+ Execute(("setenforce","0"),
+ only_if="test -f /selinux/enforce",
+ not_if="(! which getenforce ) || (which getenforce && getenforce | grep -q Disabled)",
+ sudo=True,
+ )
+
+ #directories
+ if params.has_namenode or params.dfs_type == 'HCFS':
+ Directory(params.hdfs_log_dir_prefix,
+ create_parents = True,
+ owner='root',
+ group=params.user_group,
+ mode=0775,
+ cd_access='a',
+ )
+ if params.has_namenode:
+ Directory(params.hadoop_pid_dir_prefix,
+ create_parents = True,
+ owner='root',
+ group='root',
+ cd_access='a',
+ )
+ Directory(params.hadoop_tmp_dir,
+ create_parents = True,
+ owner=params.hdfs_user,
+ cd_access='a',
+ )
+ #files
+ if params.security_enabled:
+ tc_owner = "root"
+ else:
+ tc_owner = params.hdfs_user
+
+ # if WebHDFS is not enabled we need this jar to create hadoop folders and copy tarballs to HDFS.
+ if params.sysprep_skip_copy_fast_jar_hdfs:
+ print "Skipping copying of fast-hdfs-resource.jar as host is sys prepped"
+ elif params.dfs_type == 'HCFS' or not WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
+ # for source-code of jar goto contrib/fast-hdfs-resource
+ File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
+ mode=0644,
+ content=StaticFile("fast-hdfs-resource.jar")
+ )
+
+ if os.path.exists(params.hadoop_conf_dir):
+ File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'),
+ owner=tc_owner,
+ content=Template('commons-logging.properties.j2')
+ )
+
+ health_check_template_name = "health_check"
+ File(os.path.join(params.hadoop_conf_dir, health_check_template_name),
+ owner=tc_owner,
+ content=Template(health_check_template_name + ".j2")
+ )
+
+ log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties")
+ if (params.log4j_props != None):
+ File(log4j_filename,
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))):
+ File(log4j_filename,
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ )
+
+ File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
+ owner=params.hdfs_user,
+ group=params.user_group,
+ content=InlineTemplate(params.hadoop_metrics2_properties_content)
+ )
+
+ if params.dfs_type == 'HCFS' and params.has_core_site and 'ECS_CLIENT' in params.component_list:
+ create_dirs()
+
+ create_microsoft_r_dir()
+
+
+def setup_configs():
+ """
+ Creates configs for services HDFS mapred
+ """
+ import params
+
+ if params.has_namenode or params.dfs_type == 'HCFS':
+ if os.path.exists(params.hadoop_conf_dir):
+ File(params.task_log4j_properties_location,
+ content=StaticFile("task-log4j.properties"),
+ mode=0755
+ )
+
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')):
+ File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')):
+ File(os.path.join(params.hadoop_conf_dir, 'masters'),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+ generate_include_file()
+
+
+def generate_include_file():
+ import params
+
+ if params.has_namenode and params.dfs_hosts and params.has_slaves:
+ include_hosts_list = params.slave_hosts
+ File(params.dfs_hosts,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+def create_javahome_symlink():
+ if os.path.exists("/usr/jdk/jdk1.6.0_31") and not os.path.exists("/usr/jdk64/jdk1.6.0_31"):
+ Directory("/usr/jdk64/",
+ create_parents = True,
+ )
+ Link("/usr/jdk/jdk1.6.0_31",
+ to="/usr/jdk64/jdk1.6.0_31",
+ )
+
+def create_dirs():
+ import params
+ params.HdfsResource(params.hdfs_tmp_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ mode=0777
+ )
+ params.HdfsResource(params.smoke_hdfs_user_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.smoke_user,
+ mode=params.smoke_hdfs_user_mode
+ )
+ params.HdfsResource(None,
+ action="execute"
+ )
+
+def create_microsoft_r_dir():
+ import params
+ if 'MICROSOFT_R_NODE_CLIENT' in params.component_list and params.default_fs:
+ directory = '/user/RevoShare'
+ try:
+ params.HdfsResource(directory,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ mode=0777)
+ params.HdfsResource(None, action="execute")
+ except Exception as exception:
+ Logger.warning("Could not check the existence of {0} on DFS while starting {1}, exception: {2}".format(directory, params.current_service, str(exception)))
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/commons-logging.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/commons-logging.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/commons-logging.properties.j2
new file mode 100644
index 0000000..2197ba5
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/commons-logging.properties.j2
@@ -0,0 +1,43 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+#Logging Implementation
+
+#Log4J
+org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
+
+#JDK Logger
+#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/exclude_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/exclude_hosts_list.j2
new file mode 100644
index 0000000..1adba80
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/exclude_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in hdfs_exclude_file %}
+{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
new file mode 100644
index 0000000..2f3aab6
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/hadoop-metrics2.properties.j2
@@ -0,0 +1,105 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# syntax: [prefix].[source|sink|jmx].[instance].[options]
+# See package.html for org.apache.hadoop.metrics2 for details
+
+{% if has_ganglia_server %}
+*.period=60
+
+*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
+*.sink.ganglia.period=10
+
+# default for supportsparse is false
+*.sink.ganglia.supportsparse=true
+
+.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
+.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+
+# Hook up to the server
+namenode.sink.ganglia.servers={{ganglia_server_host}}:8661
+datanode.sink.ganglia.servers={{ganglia_server_host}}:8659
+jobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662
+tasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658
+maptask.sink.ganglia.servers={{ganglia_server_host}}:8660
+reducetask.sink.ganglia.servers={{ganglia_server_host}}:8660
+resourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664
+nodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657
+historyserver.sink.ganglia.servers={{ganglia_server_host}}:8666
+journalnode.sink.ganglia.servers={{ganglia_server_host}}:8654
+nimbus.sink.ganglia.servers={{ganglia_server_host}}:8649
+supervisor.sink.ganglia.servers={{ganglia_server_host}}:8650
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
+
+{% endif %}
+
+{% if has_metric_collector %}
+
+*.period={{metrics_collection_period}}
+*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
+*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+*.sink.timeline.period={{metrics_collection_period}}
+*.sink.timeline.sendInterval={{metrics_report_interval}}000
+*.sink.timeline.slave.host.name={{hostname}}
+*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}
+*.sink.timeline.protocol={{metric_collector_protocol}}
+*.sink.timeline.port={{metric_collector_port}}
+
+# HTTPS properties
+*.sink.timeline.truststore.path = {{metric_truststore_path}}
+*.sink.timeline.truststore.type = {{metric_truststore_type}}
+*.sink.timeline.truststore.password = {{metric_truststore_password}}
+
+datanode.sink.timeline.collector.hosts={{ams_collector_hosts}}
+namenode.sink.timeline.collector.hosts={{ams_collector_hosts}}
+resourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}
+nodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}
+jobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}
+journalnode.sink.timeline.collector.hosts={{ams_collector_hosts}}
+applicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}
+
+resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue
+
+{% if is_nn_client_port_configured %}
+# Namenode rpc ports customization
+namenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}}
+{% endif %}
+{% if is_nn_dn_port_configured %}
+namenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}}
+{% endif %}
+{% if is_nn_healthcheck_port_configured %}
+namenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}}
+{% endif %}
+
+{% endif %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/health_check.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/health_check.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/health_check.j2
new file mode 100644
index 0000000..0a03d17
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/health_check.j2
@@ -0,0 +1,81 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#!/bin/bash
+#
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+err=0;
+
+function check_disks {
+
+ for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do
+ fsdev=""
+ fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`;
+ if [ -z "$fsdev" -a "$m" != "/mnt" ] ; then
+ msg_="$msg_ $m(u)"
+ else
+ msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`"
+ fi
+ done
+
+ if [ -z "$msg_" ] ; then
+ echo "disks ok" ; exit 0
+ else
+ echo "$msg_" ; exit 2
+ fi
+
+}
+
+# Run all checks
+for check in disks ; do
+ msg=`check_${check}` ;
+ if [ $? -eq 0 ] ; then
+ ok_msg="$ok_msg$msg,"
+ else
+ err_msg="$err_msg$msg,"
+ fi
+done
+
+if [ ! -z "$err_msg" ] ; then
+ echo -n "ERROR $err_msg "
+fi
+if [ ! -z "$ok_msg" ] ; then
+ echo -n "OK: $ok_msg"
+fi
+
+echo
+
+# Success!
+exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/include_hosts_list.j2
new file mode 100644
index 0000000..4a9e713
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in slave_hosts %}
+{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/topology_mappings.data.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/topology_mappings.data.j2 b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/topology_mappings.data.j2
new file mode 100644
index 0000000..15034d6
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/templates/topology_mappings.data.j2
@@ -0,0 +1,24 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+ #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+[network_topology]
+{% for host in all_hosts %}
+{% if host in slave_hosts %}
+{{host}}={{all_racks[loop.index-1]}}
+{{all_ipv4_ips[loop.index-1]}}={{all_racks[loop.index-1]}}
+{% endif %}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/3.0/kerberos.json
new file mode 100644
index 0000000..9579d0f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/kerberos.json
@@ -0,0 +1,78 @@
+{
+ "properties": {
+ "realm": "${kerberos-env/realm}",
+ "keytab_dir": "/etc/security/keytabs",
+ "additional_realms": ""
+ },
+ "identities": [
+ {
+ "name": "spnego",
+ "principal": {
+ "value": "HTTP/_HOST@${realm}",
+ "type": "service"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/spnego.service.keytab",
+ "owner": {
+ "name": "root",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ }
+ }
+ },
+ {
+ "name": "smokeuser",
+ "principal": {
+ "value": "${cluster-env/smokeuser}-${cluster_name|toLower()}@${realm}",
+ "type": "user",
+ "configuration": "cluster-env/smokeuser_principal_name",
+ "local_username": "${cluster-env/smokeuser}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/smokeuser.headless.keytab",
+ "owner": {
+ "name": "${cluster-env/smokeuser}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "cluster-env/smokeuser_keytab"
+ }
+ }
+ ],
+ "services": [
+ {
+ "name": "AMBARI",
+ "components": [
+ {
+ "name": "AMBARI_SERVER",
+ "identities": [
+ {
+ "name": "ambari-server",
+ "principal": {
+ "value": "ambari-server-${cluster_name|toLower()}@${realm}",
+ "type": "user",
+ "configuration": "cluster-env/ambari_principal_name"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/ambari.server.keytab",
+ "owner": {
+ "access": "r"
+ }
+ }
+ },
+ {
+ "name" : "ambari-server_spnego",
+ "reference" : "/spnego"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml
new file mode 100644
index 0000000..0364d41
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/metainfo.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <versions>
+ <active>true</active>
+ </versions>
+ <minJdk>1.7</minJdk>
+ <maxJdk>1.8</maxJdk>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json
new file mode 100644
index 0000000..dd87b72
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json
@@ -0,0 +1,323 @@
+{
+ "stack_features": [
+ {
+ "name": "snappy",
+ "description": "Snappy compressor/decompressor support",
+ "min_version": "2.0.0.0",
+ "max_version": "2.2.0.0"
+ },
+ {
+ "name": "lzo",
+ "description": "LZO libraries support",
+ "min_version": "2.2.1.0"
+ },
+ {
+ "name": "express_upgrade",
+ "description": "Express upgrade support",
+ "min_version": "2.1.0.0"
+ },
+ {
+ "name": "rolling_upgrade",
+ "description": "Rolling upgrade support",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "config_versioning",
+ "description": "Configurable versions support",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "datanode_non_root",
+ "description": "DataNode running as non-root support (AMBARI-7615)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "remove_ranger_hdfs_plugin_env",
+ "description": "HDFS removes Ranger env files (AMBARI-14299)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "ranger",
+ "description": "Ranger Service support",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "ranger_tagsync_component",
+ "description": "Ranger Tagsync component support (AMBARI-14383)",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "phoenix",
+ "description": "Phoenix Service support",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "nfs",
+ "description": "NFS support",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "tez_for_spark",
+ "description": "Tez dependency for Spark",
+ "min_version": "2.2.0.0",
+ "max_version": "2.3.0.0"
+ },
+ {
+ "name": "timeline_state_store",
+ "description": "Yarn application timeline-service supports state store property (AMBARI-11442)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "copy_tarball_to_hdfs",
+ "description": "Copy tarball to HDFS support (AMBARI-12113)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "spark_16plus",
+ "description": "Spark 1.6+",
+ "min_version": "2.4.0.0"
+ },
+ {
+ "name": "spark_thriftserver",
+ "description": "Spark Thrift Server",
+ "min_version": "2.3.2.0"
+ },
+ {
+ "name": "storm_kerberos",
+ "description": "Storm Kerberos support (AMBARI-7570)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "storm_ams",
+ "description": "Storm AMS integration (AMBARI-10710)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "create_kafka_broker_id",
+ "description": "Ambari should create Kafka Broker Id (AMBARI-12678)",
+ "min_version": "2.2.0.0",
+ "max_version": "2.3.0.0"
+ },
+ {
+ "name": "kafka_listeners",
+ "description": "Kafka listeners (AMBARI-10984)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "kafka_kerberos",
+ "description": "Kafka Kerberos support (AMBARI-10984)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "pig_on_tez",
+ "description": "Pig on Tez support (AMBARI-7863)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "ranger_usersync_non_root",
+ "description": "Ranger Usersync as non-root user (AMBARI-10416)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "ranger_audit_db_support",
+ "description": "Ranger Audit to DB support",
+ "min_version": "2.2.0.0",
+ "max_version": "2.5.0.0"
+ },
+ {
+ "name": "accumulo_kerberos_user_auth",
+ "description": "Accumulo Kerberos User Auth (AMBARI-10163)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "knox_versioned_data_dir",
+ "description": "Use versioned data dir for Knox (AMBARI-13164)",
+ "min_version": "2.3.2.0"
+ },
+ {
+ "name": "knox_sso_topology",
+ "description": "Knox SSO Topology support (AMBARI-13975)",
+ "min_version": "2.3.8.0"
+ },
+ {
+ "name": "atlas_rolling_upgrade",
+ "description": "Rolling upgrade support for Atlas",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "oozie_admin_user",
+ "description": "Oozie install user as an Oozie admin user (AMBARI-7976)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "oozie_create_hive_tez_configs",
+ "description": "Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "oozie_setup_shared_lib",
+ "description": "Oozie setup tools used to shared Oozie lib to HDFS (AMBARI-7240)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "oozie_host_kerberos",
+ "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)",
+ "min_version": "2.0.0.0",
+ "max_version": "2.2.0.0"
+ },
+ {
+ "name": "falcon_extensions",
+ "description": "Falcon Extension",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "hive_metastore_upgrade_schema",
+ "description": "Hive metastore upgrade schema support (AMBARI-11176)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "hive_server_interactive",
+ "description": "Hive server interactive support (AMBARI-15573)",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "hive_webhcat_specific_configs",
+ "description": "Hive webhcat specific configurations support (AMBARI-12364)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "hive_purge_table",
+ "description": "Hive purge table support (AMBARI-12260)",
+ "min_version": "2.3.0.0"
+ },
+ {
+ "name": "hive_server2_kerberized_env",
+ "description": "Hive server2 working on kerberized environment (AMBARI-13749)",
+ "min_version": "2.2.3.0",
+ "max_version": "2.2.5.0"
+ },
+ {
+ "name": "hive_env_heapsize",
+ "description": "Hive heapsize property defined in hive-env (AMBARI-12801)",
+ "min_version": "2.2.0.0"
+ },
+ {
+ "name": "ranger_kms_hsm_support",
+ "description": "Ranger KMS HSM support (AMBARI-15752)",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_log4j_support",
+ "description": "Ranger supporting log-4j properties (AMBARI-15681)",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_kerberos_support",
+ "description": "Ranger Kerberos support",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "hive_metastore_site_support",
+ "description": "Hive Metastore site support",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_usersync_password_jceks",
+ "description": "Saving Ranger Usersync credentials in jceks",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_install_infra_client",
+ "description": "Ambari Infra Service support",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "falcon_atlas_support_2_3",
+ "description": "Falcon Atlas integration support for 2.3 stack",
+ "min_version": "2.3.99.0",
+ "max_version": "2.4.0.0"
+ },
+ {
+ "name": "falcon_atlas_support",
+ "description": "Falcon Atlas integration",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "hbase_home_directory",
+ "description": "Hbase home directory in HDFS needed for HBASE backup",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "spark_livy",
+ "description": "Livy as slave component of spark",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "atlas_ranger_plugin_support",
+ "description": "Atlas Ranger plugin support",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "atlas_conf_dir_in_path",
+ "description": "Prepend the Atlas conf dir (/etc/atlas/conf) to the classpath of Storm and Falcon",
+ "min_version": "2.3.0.0",
+ "max_version": "2.4.99.99"
+ },
+ {
+ "name": "atlas_upgrade_support",
+ "description": "Atlas supports express and rolling upgrades",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "atlas_hook_support",
+ "description": "Atlas support for hooks in Hive, Storm, Falcon, and Sqoop",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_pid_support",
+ "description": "Ranger Service support pid generation AMBARI-16756",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_kms_pid_support",
+ "description": "Ranger KMS Service support pid generation",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_admin_password_change",
+ "description": "Allow ranger admin credentials to be specified during cluster creation (AMBARI-17000)",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "storm_metrics_apache_classes",
+ "description": "Metrics sink for Storm that uses Apache class names",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "spark_java_opts_support",
+ "description": "Allow Spark to generate java-opts file",
+ "min_version": "2.2.0.0",
+ "max_version": "2.4.0.0"
+ },
+ {
+ "name": "atlas_hbase_setup",
+ "description": "Use script to create Atlas tables in Hbase and set permissions for Atlas user.",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "ranger_hive_plugin_jdbc_url",
+ "description": "Handle Ranger hive repo config jdbc url change for stack 2.5 (AMBARI-18386)",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "zkfc_version_advertised",
+ "description": "ZKFC advertise version",
+ "min_version": "2.5.0.0"
+ },
+ {
+ "name": "phoenix_core_hdfs_site_required",
+ "description": "HDFS and CORE site required for Phoenix",
+ "max_version": "2.5.9.9"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_tools.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_tools.json b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_tools.json
new file mode 100644
index 0000000..d1aab4b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_tools.json
@@ -0,0 +1,4 @@
+{
+ "stack_selector": ["hdp-select", "/usr/bin/hdp-select", "hdp-select"],
+ "conf_selector": ["conf-select", "/usr/bin/conf-select", "conf-select"]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/repos/repoinfo.xml
new file mode 100644
index 0000000..5145064
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/repos/repoinfo.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <latest>http://s3.amazonaws.com/dev.hortonworks.com/HDP/hdp_urlinfo.json</latest>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/3.x/updates/3.0.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="redhat7">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos7/3.x/updates/3.0.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos7</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="suse11">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11sp3/3.x/updates/3.0.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/suse11sp3</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="suse12">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/sles12/3.x/updates/3.0.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/suse11sp3</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="ubuntu12">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/ubuntu12/3.x/updates/2.3.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/ubuntu12</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="debian7">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/debian7/3.x/updates/2.3.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/debian6</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="ubuntu14">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/ubuntu14/3.x/updates/2.3.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/ubuntu12</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+ <os family="ubuntu16">
+ <repo>
+ <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/ubuntu14/3.x/updates/3.0.0.0</baseurl>
+ <repoid>HDP-3.0</repoid>
+ <reponame>HDP</reponame>
+ <unique>true</unique>
+ </repo>
+ <repo>
+ <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/ubuntu12</baseurl>
+ <repoid>HDP-UTILS-1.1.0.21</repoid>
+ <reponame>HDP-UTILS</reponame>
+ <unique>false</unique>
+ </repo>
+ </os>
+</reposinfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/widgets.json b/ambari-server/src/main/resources/stacks/HDP/3.0/widgets.json
new file mode 100644
index 0000000..3176354
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/widgets.json
@@ -0,0 +1,95 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_system_heatmap",
+ "display_name": "Heatmaps",
+ "section_name": "SYSTEM_HEATMAPS",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "Host Disk Space Used %",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "disk_free",
+ "metric_path": "metrics/disk/disk_free",
+ "service_name": "STACK"
+ },
+ {
+ "name": "disk_total",
+ "metric_path": "metrics/disk/disk_total",
+ "service_name": "STACK"
+ }
+ ],
+ "values": [
+ {
+ "name": "Host Disk Space Used %",
+ "value": "${((disk_total-disk_free)/disk_total)*100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Host Memory Used %",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "mem_total",
+ "metric_path": "metrics/memory/mem_total",
+ "service_name": "STACK"
+ },
+ {
+ "name": "mem_free",
+ "metric_path": "metrics/memory/mem_free",
+ "service_name": "STACK"
+ },
+ {
+ "name": "mem_cached",
+ "metric_path": "metrics/memory/mem_cached",
+ "service_name": "STACK"
+ }
+ ],
+ "values": [
+ {
+ "name": "Host Memory Used %",
+ "value": "${((mem_total-mem_free-mem_cached)/mem_total)*100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Host CPU Wait IO %",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "cpu_wio",
+ "metric_path": "metrics/cpu/cpu_wio",
+ "service_name": "STACK"
+ }
+ ],
+ "values": [
+ {
+ "name": "Host Memory Used %",
+ "value": "${cpu_wio*100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ }
+ ]
+ }
+ ]
+}
[50/50] [abbrv] ambari git commit: Merge branch 'trunk' into
branch-dev-patch-upgrade
Posted by nc...@apache.org.
Merge branch 'trunk' into branch-dev-patch-upgrade
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/90cb3a4c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/90cb3a4c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/90cb3a4c
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 90cb3a4ce92c4cfb57356073d1fa603e2403f632
Parents: 06327e6 234838f
Author: Nate Cole <nc...@hortonworks.com>
Authored: Wed Dec 7 09:59:40 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Wed Dec 7 09:59:40 2016 -0500
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 3 +
.../ambari_agent/CustomServiceOrchestrator.py | 120 +++
.../src/main/python/ambari_agent/HostInfo.py | 12 +-
ambari-agent/src/packages/tarball/all.xml | 30 +
.../ambari_commons/ambari_metrics_helper.py | 26 +-
.../libraries/functions/constants.py | 1 +
.../libraries/functions/copy_tarball.py | 2 +-
.../dynamic_variable_interpretation.py | 2 +-
.../libraries/functions/setup_atlas_hook.py | 47 +-
.../timeline/AbstractTimelineMetricsSink.java | 40 +-
.../availability/MetricCollectorHAHelper.java | 1 -
.../availability/MetricCollectorHATest.java | 1 -
.../src/main/python/core/config_reader.py | 2 +-
.../timeline/HBaseTimelineMetricStore.java | 18 +-
.../timeline/TimelineMetricConfiguration.java | 14 +
.../aggregators/AbstractTimelineAggregator.java | 6 +-
.../MetricCollectorHAController.java | 4 +-
.../TestApplicationHistoryServer.java | 14 +-
.../MetricCollectorHAControllerTest.java | 4 +-
ambari-project/pom.xml | 41 +
ambari-server/checkstyle.xml | 17 +
ambari-server/pom.xml | 4 +
.../ambari/server/agent/ExecutionCommand.java | 28 +
.../ambari/server/agent/HeartBeatHandler.java | 2 +-
.../AmbariManagementControllerImpl.java | 5 +
.../internal/AlertTargetResourceProvider.java | 2 +-
.../BlueprintConfigurationProcessor.java | 3 +-
.../ClusterStackVersionResourceProvider.java | 4 +-
.../internal/UpgradeResourceProvider.java | 2 +-
.../apache/ambari/server/orm/dao/AlertsDAO.java | 6 +-
.../ambari/server/orm/entities/GroupEntity.java | 5 +-
.../server/security/authorization/Users.java | 10 +-
.../apache/ambari/server/state/ServiceImpl.java | 2 +-
.../server/state/cluster/ClustersImpl.java | 2 +-
.../ambari/server/state/host/HostImpl.java | 2 +-
.../services/RetryUpgradeActionService.java | 2 +-
.../ambari/server/state/stack/UpgradePack.java | 19 +-
.../svccomphost/ServiceComponentHostImpl.java | 2 +-
.../ambari/server/topology/TopologyManager.java | 19 +-
.../server/upgrade/AbstractUpgradeCatalog.java | 25 +-
.../server/upgrade/UpgradeCatalog240.java | 4 +-
.../server/upgrade/UpgradeCatalog250.java | 30 +
.../src/main/python/ambari_server_main.py | 8 +-
.../AMBARI_INFRA/0.1.0/metainfo.xml | 1 +
.../0.1.0/package/scripts/setup_infra_solr.py | 3 +-
.../0.1.0/configuration/ams-env.xml | 2 +-
.../0.1.0/configuration/ams-site.xml | 12 +
.../0.1.0/package/scripts/metrics_collector.py | 2 +
.../package/scripts/metrics_grafana_util.py | 7 +-
.../0.1.0/package/scripts/params.py | 22 +-
.../0.1.0/package/scripts/service_check.py | 2 +-
.../0.5.0.2.1/configuration/falcon-env.xml | 17 +
.../FALCON/0.5.0.2.1/metainfo.xml | 7 +
.../FALCON/0.5.0.2.1/package/scripts/falcon.py | 2 +-
.../0.5.0.2.1/package/scripts/params_linux.py | 5 +-
.../FALCON/0.5.0.2.1/quicklinks/quicklinks.json | 35 +
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 17 +
.../HIVE/0.12.0.2.0/package/scripts/hcat.py | 2 +-
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 2 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 5 +-
.../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +-
.../configuration/logsearch-properties.xml | 2 +-
.../LOGSEARCH/0.5.0/metainfo.xml | 1 +
.../0.5.0/package/scripts/setup_logsearch.py | 3 +-
.../RANGER/0.4.0/package/scripts/params.py | 1 +
.../0.4.0/package/scripts/ranger_admin.py | 36 +-
.../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 17 +
.../1.4.4.2.0/configuration/sqoop-site.xml | 6 +
.../1.4.4.2.0/package/scripts/params_linux.py | 5 +-
.../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 2 +-
.../STORM/0.10.0/configuration/storm-env.xml | 17 +
.../STORM/0.9.1/package/scripts/params_linux.py | 6 +-
.../STORM/0.9.1/package/scripts/storm.py | 2 +-
.../scripts/post-user-creation-hook.sh | 7 +
.../HDP/2.0.6/properties/stack_features.json | 5 +
.../services/HIVE/configuration/hive-site.xml | 4 +
.../services/STORM/configuration/storm-site.xml | 4 +
.../stacks/HDP/2.3/services/stack_advisor.py | 61 +-
.../services/HDFS/configuration/hadoop-env.xml | 176 ++++
.../stacks/HDP/2.5/services/stack_advisor.py | 6 -
.../HDP/2.6/services/ACCUMULO/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/FALCON/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/FLUME/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/HBASE/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/HDFS/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/HIVE/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/KAFKA/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/KNOX/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/MAHOUT/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/OOZIE/metainfo.xml | 1 +
.../stacks/HDP/2.6/services/PIG/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/RANGER/metainfo.xml | 2 +-
.../HDP/2.6/services/RANGER_KMS/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/SLIDER/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/SPARK/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/SPARK2/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/SQOOP/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/STORM/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/TEZ/metainfo.xml | 2 +-
.../stacks/HDP/2.6/services/YARN/metainfo.xml | 4 +-
.../HDP/2.6/services/ZOOKEEPER/metainfo.xml | 2 +-
.../HDP/3.0/configuration/cluster-env.xml | 293 +++++++
.../HDP/3.0/hooks/after-INSTALL/scripts/hook.py | 37 +
.../3.0/hooks/after-INSTALL/scripts/params.py | 97 +++
.../scripts/shared_initialization.py | 111 +++
.../hooks/before-ANY/files/changeToSecureUid.sh | 53 ++
.../HDP/3.0/hooks/before-ANY/scripts/hook.py | 36 +
.../HDP/3.0/hooks/before-ANY/scripts/params.py | 231 ++++++
.../before-ANY/scripts/shared_initialization.py | 226 +++++
.../3.0/hooks/before-INSTALL/scripts/hook.py | 37 +
.../3.0/hooks/before-INSTALL/scripts/params.py | 113 +++
.../scripts/repo_initialization.py | 68 ++
.../scripts/shared_initialization.py | 37 +
.../3.0/hooks/before-RESTART/scripts/hook.py | 29 +
.../hooks/before-START/files/checkForFormat.sh | 65 ++
.../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 19285850 bytes
.../before-START/files/task-log4j.properties | 134 +++
.../hooks/before-START/files/topology_script.py | 66 ++
.../HDP/3.0/hooks/before-START/scripts/hook.py | 39 +
.../3.0/hooks/before-START/scripts/params.py | 326 ++++++++
.../before-START/scripts/rack_awareness.py | 47 ++
.../scripts/shared_initialization.py | 191 +++++
.../templates/commons-logging.properties.j2 | 43 +
.../templates/exclude_hosts_list.j2 | 21 +
.../templates/hadoop-metrics2.properties.j2 | 105 +++
.../before-START/templates/health_check.j2 | 81 ++
.../templates/include_hosts_list.j2 | 21 +
.../templates/topology_mappings.data.j2 | 24 +
.../main/resources/stacks/HDP/3.0/kerberos.json | 78 ++
.../main/resources/stacks/HDP/3.0/metainfo.xml | 24 +
.../HDP/3.0/properties/stack_features.json | 323 ++++++++
.../stacks/HDP/3.0/properties/stack_tools.json | 4 +
.../resources/stacks/HDP/3.0/repos/repoinfo.xml | 132 +++
.../services/HDFS/configuration/core-site.xml | 56 ++
.../services/HDFS/configuration/hadoop-env.xml | 200 +++++
.../services/HDFS/configuration/hdfs-log4j.xml | 226 +++++
.../services/HDFS/configuration/hdfs-site.xml | 153 ++++
.../HDFS/configuration/ranger-hdfs-audit.xml | 217 +++++
.../ranger-hdfs-plugin-properties.xml | 98 +++
.../configuration/ranger-hdfs-policymgr-ssl.xml | 67 ++
.../HDFS/configuration/ranger-hdfs-security.xml | 65 ++
.../services/HDFS/configuration/widgets.json | 649 +++++++++++++++
.../stacks/HDP/3.0/services/HDFS/kerberos.json | 246 ++++++
.../stacks/HDP/3.0/services/HDFS/metainfo.xml | 190 +++++
.../services/HDFS/quicklinks/quicklinks.json | 80 ++
.../HDP/3.0/services/HDFS/themes/theme.json | 179 ++++
.../HDP/3.0/services/YARN/YARN_widgets.json | 670 +++++++++++++++
.../YARN/configuration-mapred/mapred-env.xml | 51 ++
.../YARN/configuration-mapred/mapred-site.xml | 134 +++
.../YARN/configuration/capacity-scheduler.xml | 71 ++
.../YARN/configuration/ranger-yarn-audit.xml | 177 ++++
.../ranger-yarn-plugin-properties.xml | 82 ++
.../configuration/ranger-yarn-policymgr-ssl.xml | 66 ++
.../YARN/configuration/ranger-yarn-security.xml | 58 ++
.../services/YARN/configuration/yarn-env.xml | 200 +++++
.../services/YARN/configuration/yarn-log4j.xml | 103 +++
.../services/YARN/configuration/yarn-site.xml | 814 +++++++++++++++++++
.../stacks/HDP/3.0/services/YARN/kerberos.json | 278 +++++++
.../stacks/HDP/3.0/services/YARN/metainfo.xml | 173 ++++
.../YARN/quicklinks-mapred/quicklinks.json | 80 ++
.../services/YARN/quicklinks/quicklinks.json | 80 ++
.../3.0/services/YARN/themes-mapred/theme.json | 132 +++
.../HDP/3.0/services/YARN/themes/theme.json | 250 ++++++
.../HDP/3.0/services/ZOOKEEPER/metainfo.xml | 54 ++
.../main/resources/stacks/HDP/3.0/widgets.json | 95 +++
.../2.1/hooks/before-START/scripts/params.py | 2 +-
.../HAPPY/configuration/happy-alert-config.xml | 5 +
.../HBASE/configuration/hbase-alert-config.xml | 5 +
.../HDFS/configuration/hdfs-alert-config.xml | 5 +
.../configuration/sleepy-alert-config.xml | 5 +
.../SNOW/configuration/snow-alert-config.xml | 5 +
.../YARN/configuration/yarn-alert-config.xml | 5 +
.../ZOOKEEPER/configuration/zk-alert-config.xml | 5 +
.../server/agent/TestHeartbeatHandler.java | 28 +-
.../BlueprintConfigurationProcessorTest.java | 11 +
.../ldap/AmbariLdapDataPopulatorTest.java | 3 +-
.../ambari/server/state/UpgradeHelperTest.java | 39 +-
.../server/upgrade/UpgradeCatalog210Test.java | 2 +
.../server/upgrade/UpgradeCatalog250Test.java | 93 +++
.../stacks/2.3/common/test_stack_advisor.py | 57 +-
.../stacks/2.4/AMBARI_INFRA/test_infra_solr.py | 2 +-
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 2 +-
.../stacks/2.6/RANGER/test_ranger_admin.py | 504 ++++++++++++
.../2.6/configs/ranger-admin-default.json | 704 ++++++++++++++++
.../2.6/configs/ranger-admin-secured.json | 773 ++++++++++++++++++
.../app/controllers/global/update_controller.js | 5 +-
ambari-web/app/controllers/installer.js | 3 +-
.../journalNode/step4_controller.js | 6 +-
.../journalNode/wizard_controller.js | 28 +-
.../main/admin/stack_and_upgrade_controller.js | 13 +-
.../app/controllers/wizard/step1_controller.js | 21 +-
ambari-web/app/messages.js | 3 -
.../common/widgets/export_metrics_mixin.js | 28 +-
.../main/host/details/actions/check_host.js | 3 +-
.../configs/objects/service_config_property.js | 7 -
.../app/routes/manage_journalnode_routes.js | 4 +-
.../app/styles/theme/bootstrap-ambari.css | 29 +-
ambari-web/app/styles/wizard.less | 19 +-
.../common/configs/notifications_configs.hbs | 2 +-
.../templates/common/host_progress_popup.hbs | 8 +-
.../highAvailability/journalNode/wizard.hbs | 13 +-
.../admin/stack_upgrade/upgrade_options.hbs | 58 +-
ambari-web/app/templates/wizard/step1.hbs | 2 +-
ambari-web/app/templates/wizard/step8.hbs | 2 +-
.../notification_configs_view.js | 18 +-
.../configs/service_configs_by_category_view.js | 9 +-
.../widgets/slider_config_widget_view.js | 7 +-
ambari-web/app/views/common/controls_view.js | 3 +-
.../highAvailability/journalNode/step2_view.js | 5 +-
.../admin/stack_upgrade/upgrade_wizard_view.js | 6 +-
ambari-web/app/views/main/host/menu.js | 4 +-
ambari-web/app/views/wizard/step1_view.js | 21 +-
ambari-web/test/controllers/installer_test.js | 13 +-
.../widgets/slider_config_widget_view_test.js | 23 -
.../test/views/common/controls_view_test.js | 74 +-
.../stack_upgrade/upgrade_wizard_view_test.js | 2 +-
.../test/views/main/dashboard/widgets_test.js | 2 +-
contrib/utils/perf/deploy-gce-perf-cluster.py | 10 +-
contrib/views/ambari-views-package/pom.xml | 4 +-
.../src/main/resources/view.xml | 23 +-
contrib/views/files/src/main/resources/view.xml | 7 +
.../views/hive-next/src/main/resources/view.xml | 7 +
contrib/views/tez/src/main/resources/view.xml | 7 +
pom.xml | 1 +
utility/pom.xml | 33 +-
...AvoidTransactionalOnPrivateMethodsCheck.java | 55 ++
.../src/main/resources/checkstyle_packages.xml | 15 +
...dTransactionalOnPrivateMethodsCheckTest.java | 49 ++
.../InputTransactionalOnPrivateMethods.java | 46 ++
229 files changed, 12765 insertions(+), 414 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/90cb3a4c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/90cb3a4c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/90cb3a4c/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
[34/50] [abbrv] ambari git commit: AMBARI-19101 : Merge AMS service
check support for HA from trunk. (dsen, avijayan via avijayan)
Posted by nc...@apache.org.
AMBARI-19101 : Merge AMS service check support for HA from trunk. (dsen, avijayan via avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9a938e6c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9a938e6c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9a938e6c
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 9a938e6cc27811057895d495f700a8e874c69a10
Parents: 6418861
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Dec 6 12:50:45 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Dec 6 12:50:54 2016 -0800
----------------------------------------------------------------------
.../ambari_commons/ambari_metrics_helper.py | 26 ++++++++++----------
.../package/scripts/metrics_grafana_util.py | 2 +-
.../0.1.0/package/scripts/service_check.py | 2 +-
.../2.1/hooks/before-START/scripts/params.py | 2 +-
4 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a938e6c/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py b/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py
index f6f4068..bfc786c 100644
--- a/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py
+++ b/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py
@@ -29,15 +29,15 @@ def select_metric_collector_for_sink(sink_name):
# TODO check '*' sink_name
all_collectors_string = get_metric_collectors_from_properties_file(sink_name)
- if all_collectors_string:
- all_collectors_list = all_collectors_string.split(',')
- return select_metric_collector_hosts_from_hostnames(all_collectors_list)
+ return select_metric_collector_hosts_from_hostnames(all_collectors_string)
+
+def select_metric_collector_hosts_from_hostnames(comma_separated_hosts):
+ if comma_separated_hosts:
+ hosts = comma_separated_hosts.split(',')
+ return get_random_host(hosts)
else:
return 'localhost'
-def select_metric_collector_hosts_from_hostnames(hosts):
- return get_random_host(hosts)
-
def get_random_host(hosts):
return random.choice(hosts)
@@ -53,10 +53,10 @@ def load_properties_from_file(filepath, sep='=', comment_char='#'):
props = {}
with open(filepath, "rt") as f:
for line in f:
- l = line.strip()
- if l and not l.startswith(comment_char):
- key_value = l.split(sep)
- key = key_value[0].strip()
- value = sep.join(key_value[1:]).strip('" \t')
- props[key] = value
- return props
+ l = line.strip()
+ if l and not l.startswith(comment_char):
+ key_value = l.split(sep)
+ key = key_value[0].strip()
+ value = sep.join(key_value[1:]).strip('" \t')
+ props[key] = value
+ return props
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a938e6c/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
index 41bd2b4..75740f8 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py
@@ -215,7 +215,7 @@ def create_ams_datasource():
Create AMS datasource in Grafana, if exsists make sure the collector url is accurate
"""
Logger.info("Trying to find working metric collector")
- results = execute_in_parallel(do_ams_collector_post, params.ams_collector_hosts, params)
+ results = execute_in_parallel(do_ams_collector_post, params.ams_collector_hosts.split(','), params)
new_datasource_host = ""
for host in params.ams_collector_hosts:
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a938e6c/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
index b4e9b8f..305fe73 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/service_check.py
@@ -144,7 +144,7 @@ class AMSServiceCheck(Script):
Logger.info("Ambari Metrics service check was started.")
env.set_params(params)
- results = execute_in_parallel(self.service_check_for_single_host, params.ams_collector_hosts, params)
+ results = execute_in_parallel(self.service_check_for_single_host, params.ams_collector_hosts.split(','), params)
for host in str(params.ams_collector_hosts).split(","):
if host in results:
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a938e6c/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
index a22eb90..1058c75 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/before-START/scripts/params.py
@@ -29,7 +29,7 @@ if has_metric_collector:
'metrics_collector_vip_host' in config['configurations']['cluster-env']:
metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
else:
- metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts.split(","))
+ metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts)
if 'cluster-env' in config['configurations'] and \
'metrics_collector_vip_port' in config['configurations']['cluster-env']:
metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
[46/50] [abbrv] ambari git commit: AMBARI-19108 Perf: Web UI can't
load json with host check info. (atkach)
Posted by nc...@apache.org.
AMBARI-19108 Perf: Web UI can't load json with host check info. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/165c4d69
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/165c4d69
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/165c4d69
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 165c4d692b4a76e2911027ac3f06d1948fb696e7
Parents: 304a5bf
Author: Andrii Tkach <at...@apache.org>
Authored: Tue Dec 6 20:17:35 2016 +0200
Committer: Andrii Tkach <at...@apache.org>
Committed: Wed Dec 7 12:47:42 2016 +0200
----------------------------------------------------------------------
ambari-web/app/mixins/main/host/details/actions/check_host.js | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/165c4d69/ambari-web/app/mixins/main/host/details/actions/check_host.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/main/host/details/actions/check_host.js b/ambari-web/app/mixins/main/host/details/actions/check_host.js
index 532353f..faf8ef1 100644
--- a/ambari-web/app/mixins/main/host/details/actions/check_host.js
+++ b/ambari-web/app/mixins/main/host/details/actions/check_host.js
@@ -193,7 +193,8 @@ App.CheckHostMixin = Em.Mixin.create({
*/
getHostCheckTasksSuccess: function (data) {
if (!data) {
- return;
+ //if resolution host check has corrupted data then skip it
+ return this.getGeneralHostCheck();
}
if (["FAILED", "COMPLETED", "TIMEDOUT"].contains(data.Requests.request_status)) {
if (data.Requests.inputs.indexOf("last_agent_env_check") != -1) {
[15/50] [abbrv] ambari git commit: AMBARI-18065. Change Ranger Admin
start script to setup db for ranger (magyari_sandor)
Posted by nc...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
new file mode 100644
index 0000000..1d5adff
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
@@ -0,0 +1,704 @@
+{
+ "localComponents": [
+ "NAMENODE",
+ "SECONDARY_NAMENODE",
+ "ZOOKEEPER_SERVER",
+ "DATANODE",
+ "HDFS_CLIENT",
+ "ZOOKEEPER_CLIENT",
+ "RANGER_USERSYNC",
+ "RANGER_ADMIN",
+ "RANGER_TAGSYNC",
+ "LOGSEARCH_SOLR",
+ "LOGSEARCH_SOLR_CLIENT"
+ ],
+ "configuration_attributes": {
+ "ranger-hdfs-audit": {},
+ "ssl-client": {},
+ "ranger-admin-site": {},
+ "ranger-hdfs-policymgr-ssl": {},
+ "tagsync-application-properties": {},
+ "ranger-env": {},
+ "usersync-log4j": {},
+ "admin-properties": {},
+ "ranger-ugsync-site": {},
+ "hdfs-site": {
+ "final": {
+ "dfs.datanode.data.dir": "true",
+ "dfs.namenode.http-address": "true",
+ "dfs.datanode.failed.volumes.tolerated": "true",
+ "dfs.support.append": "true",
+ "dfs.namenode.name.dir": "true",
+ "dfs.webhdfs.enabled": "true"
+ }
+ },
+ "ranger-tagsync-site": {},
+ "zoo.cfg": {},
+ "hadoop-policy": {},
+ "hdfs-log4j": {},
+ "ranger-hdfs-plugin-properties": {},
+ "core-site": {
+ "final": {
+ "fs.defaultFS": "true"
+ }
+ },
+ "hadoop-env": {},
+ "zookeeper-log4j": {},
+ "ssl-server": {},
+ "ranger-site": {},
+ "admin-log4j": {},
+ "tagsync-log4j": {},
+ "ranger-hdfs-security": {},
+ "usersync-properties": {},
+ "zookeeper-env": {},
+ "infra-solr-env": {},
+ "infra-solr-client-log4j": {},
+ "cluster-env": {}
+ },
+ "public_hostname": "c6401.ambari.apache.org",
+ "commandId": "11-0",
+ "hostname": "c6401.ambari.apache.org",
+ "kerberosCommandParams": [],
+ "serviceName": "RANGER",
+ "role": "RANGER_ADMIN",
+ "forceRefreshConfigTagsBeforeExecution": [],
+ "requestId": 11,
+ "agentConfigParams": {
+ "agent": {
+ "parallel_execution": 0
+ }
+ },
+ "clusterName": "c1",
+ "commandType": "EXECUTION_COMMAND",
+ "taskId": 31,
+ "roleParams": {},
+ "configurationTags": {
+ "ranger-hdfs-audit": {
+ "tag": "version1466705299922"
+ },
+ "ssl-client": {
+ "tag": "version1"
+ },
+ "ranger-admin-site": {
+ "tag": "version1466705299949"
+ },
+ "ranger-hdfs-policymgr-ssl": {
+ "tag": "version1466705299922"
+ },
+ "tagsync-application-properties": {
+ "tag": "version1466705299949"
+ },
+ "ranger-env": {
+ "tag": "version1466705299949"
+ },
+ "usersync-log4j": {
+ "tag": "version1466705299949"
+ },
+ "admin-properties": {
+ "tag": "version1466705299949"
+ },
+ "ranger-ugsync-site": {
+ "tag": "version1466705299949"
+ },
+ "hdfs-site": {
+ "tag": "version1"
+ },
+ "ranger-tagsync-site": {
+ "tag": "version1466705299949"
+ },
+ "zoo.cfg": {
+ "tag": "version1"
+ },
+ "hadoop-policy": {
+ "tag": "version1"
+ },
+ "hdfs-log4j": {
+ "tag": "version1"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "tag": "version1466705299922"
+ },
+ "core-site": {
+ "tag": "version1"
+ },
+ "hadoop-env": {
+ "tag": "version1"
+ },
+ "zookeeper-log4j": {
+ "tag": "version1"
+ },
+ "ssl-server": {
+ "tag": "version1"
+ },
+ "ranger-site": {
+ "tag": "version1466705299949"
+ },
+ "admin-log4j": {
+ "tag": "version1466705299949"
+ },
+ "tagsync-log4j": {
+ "tag": "version1466705299949"
+ },
+ "ranger-hdfs-security": {
+ "tag": "version1466705299922"
+ },
+ "usersync-properties": {
+ "tag": "version1466705299949"
+ },
+ "zookeeper-env": {
+ "tag": "version1"
+ },
+ "infra-solr-env": {
+ "tag": "version1467098537360"
+ },
+ "infra-solr-client-log4j": {
+ "tag": "version1467096917836"
+ },
+ "cluster-env": {
+ "tag": "version1"
+ }
+ },
+ "roleCommand": "START",
+ "hostLevelParams": {
+ "agent_stack_retry_on_unavailability": "false",
+ "stack_name": "HDP",
+ "package_version": "2_5_0_0_*",
+ "custom_mysql_jdbc_name": "mysql-connector-java.jar",
+ "previous_custom_mysql_jdbc_name": "mysql-connector-java-old.jar",
+ "host_sys_prepped": "false",
+ "ambari_db_rca_username": "mapred",
+ "current_version": "2.6.0.0-801",
+ "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar",
+ "agent_stack_retry_count": "5",
+ "stack_version": "2.6",
+ "jdk_name": "jdk-8u60-linux-x64.tar.gz",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "repository_version_id": "1",
+ "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
+ "not_managed_hdfs_path_list": "[\"/tmp\"]",
+ "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
+ "java_version": "8",
+ "repo_info": "[{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.5.0.0-801\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.5\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0\",\"latestBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.5.0.0-801\",\"baseSaved\":true},{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.21\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"baseSaved\":true}]",
+ "package_list": "[{\"name\":\"ranger_${stack_version}-admin\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-usersync\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-tagsync\",\"condition\":\"should_install_ranger_tagsync\",\"skipUpgrade\":false},{\"name\":\"ambari-logsearch-solr-client\",\"condition\":\"should_install_logsearch_solr_client\",\"skipUpgrade\":false}]",
+ "db_name": "ambari",
+ "group_list": "[\"ranger\",\"hadoop\",\"users\"]",
+ "agentCacheDir": "/var/lib/ambari-agent/cache",
+ "ambari_db_rca_password": "mapred",
+ "jce_name": "jce_policy-8.zip",
+ "oracle_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//ojdbc6.jar",
+ "db_driver_filename": "mysql-connector-java.jar",
+ "user_list": "[\"zookeeper\",\"ambari-qa\",\"ranger\",\"hdfs\"]",
+ "clientsToUpdateConfigs": "[\"*\"]"
+ },
+ "commandParams": {
+ "service_package_folder": "common-services/RANGER/0.4.0/package",
+ "script": "scripts/ranger_admin.py",
+ "hooks_folder": "HDP/2.0.6/hooks",
+ "version": "2.6.0.0-801",
+ "max_duration_for_retries": "0",
+ "command_retry_enabled": "false",
+ "command_timeout": "600",
+ "script_type": "PYTHON"
+ },
+ "forceRefreshConfigTags": [],
+ "stageId": 0,
+ "clusterHostInfo": {
+ "snamenode_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "ambari_server_use_ssl": [
+ "false"
+ ],
+ "all_ping_ports": [
+ "8670"
+ ],
+ "ranger_tagsync_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "ranger_usersync_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "all_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "slave_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "namenode_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "ambari_server_port": [
+ "8080"
+ ],
+ "ranger_admin_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "all_racks": [
+ "/default-rack"
+ ],
+ "all_ipv4_ips": [
+ "172.22.83.73"
+ ],
+ "ambari_server_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "zookeeper_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "infra_solr_hosts": [
+ "c6401.ambari.apache.org"
+ ]
+ },
+ "configurations": {
+ "ranger-hdfs-audit": {
+ "xasecure.audit.destination.solr.zookeepers": "NONE",
+ "xasecure.audit.destination.solr.urls": "",
+ "xasecure.audit.destination.solr.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/solr/spool",
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/hdfs/spool",
+ "xasecure.audit.destination.hdfs": "true",
+ "xasecure.audit.destination.solr": "false",
+ "xasecure.audit.provider.summary.enabled": "false",
+ "xasecure.audit.destination.hdfs.dir": "hdfs://c6401.ambari.apache.org:8020/ranger/audit",
+ "xasecure.audit.is.enabled": "true"
+ },
+ "ssl-client": {
+ "ssl.client.truststore.reload.interval": "10000",
+ "ssl.client.keystore.password": "bigdata",
+ "ssl.client.truststore.type": "jks",
+ "ssl.client.keystore.location": "/etc/security/clientKeys/keystore.jks",
+ "ssl.client.truststore.location": "/etc/security/clientKeys/all.jks",
+ "ssl.client.truststore.password": "bigdata",
+ "ssl.client.keystore.type": "jks"
+ },
+ "ranger-admin-site": {
+ "ranger.admin.kerberos.cookie.domain": "",
+ "ranger.kms.service.user.hdfs": "hdfs",
+ "ranger.spnego.kerberos.principal": "",
+ "ranger.ldap.ad.url": "{{ranger_ug_ldap_url}}",
+ "ranger.plugins.hive.serviceuser": "hive",
+ "ranger.lookup.kerberos.keytab": "",
+ "ranger.plugins.kms.serviceuser": "kms",
+ "ranger.service.https.attrib.ssl.enabled": "false",
+ "ranger.sso.browser.useragent": "Mozilla,chrome",
+ "ranger.jpa.jdbc.url": "jdbc:mysql://c6401.ambari.apache.org:3306/ranger01",
+ "ranger.plugins.hbase.serviceuser": "hbase",
+ "ranger.plugins.hdfs.serviceuser": "hdfs",
+ "ranger.ldap.group.searchbase": "{{ranger_ug_ldap_group_searchbase}}",
+ "ranger.ldap.user.dnpattern": "uid={0},ou=users,dc=xasecure,dc=net",
+ "ranger.plugins.knox.serviceuser": "knox",
+ "ranger.ldap.base.dn": "dc=example,dc=com",
+ "ranger.sso.publicKey": "",
+ "ranger.admin.kerberos.cookie.path": "/",
+ "ranger.service.https.attrib.clientAuth": "want",
+ "ranger.jpa.jdbc.user": "{{ranger_db_user}}",
+ "ranger.ldap.ad.user.searchfilter": "(sAMAccountName={0})",
+ "ranger.ldap.group.roleattribute": "cn",
+ "ranger.plugins.kafka.serviceuser": "kafka",
+ "ranger.admin.kerberos.principal": "",
+ "ranger.ldap.ad.bind.dn": "{{ranger_ug_ldap_bind_dn}}",
+ "ranger.credential.provider.path": "/etc/ranger/admin/rangeradmin.jceks",
+ "ranger.ldap.referral": "ignore",
+ "ranger.service.http.port": "6080",
+ "ranger.ldap.user.searchfilter": "(uid={0})",
+ "ranger.plugins.atlas.serviceuser": "atlas",
+ "ranger.truststore.password": "changeit",
+ "ranger.ldap.bind.password": "{{ranger_usersync_ldap_ldapbindpassword}}",
+ "ranger.audit.solr.password": "NONE",
+ "ranger.audit.solr.zookeepers": "c6401.ambari.apache.org:2181/ambari-solr",
+ "ranger.lookup.kerberos.principal": "",
+ "ranger.service.https.port": "6182",
+ "ranger.plugins.storm.serviceuser": "storm",
+ "ranger.externalurl": "{{ranger_external_url}}",
+ "ranger.truststore.file": "/etc/ranger/admin/conf/ranger-admin-keystore.jks",
+ "ranger.kms.service.user.hive": "",
+ "ranger.https.attrib.keystore.file": "/etc/ranger/admin/conf/ranger-admin-keystore.jks",
+ "ranger.jpa.jdbc.dialect": "{{jdbc_dialect}}",
+ "ranger.service.host": "{{ranger_host}}",
+ "ranger.service.https.attrib.keystore.keyalias": "rangeradmin",
+ "ranger.service.https.attrib.keystore.pass": "xasecure",
+ "ranger.unixauth.remote.login.enabled": "true",
+ "ranger.jpa.jdbc.credential.alias": "rangeradmin",
+ "ranger.ldap.ad.bind.password": "{{ranger_usersync_ldap_ldapbindpassword}}",
+ "ranger.audit.solr.username": "ranger_solr",
+ "ranger.sso.enabled": "false",
+ "ranger.audit.solr.urls": "",
+ "ranger.ldap.ad.domain": "",
+ "ranger.plugins.yarn.serviceuser": "yarn",
+ "ranger.audit.source.type": "solr",
+ "ranger.ldap.bind.dn": "{{ranger_ug_ldap_bind_dn}}",
+ "ranger.ldap.url": "{{ranger_ug_ldap_url}}",
+ "ranger.authentication.method": "UNIX",
+ "ranger.service.http.enabled": "true",
+ "ranger.ldap.group.searchfilter": "{{ranger_ug_ldap_group_searchfilter}}",
+ "ranger.ldap.ad.referral": "ignore",
+ "ranger.ldap.ad.base.dn": "dc=example,dc=com",
+ "ranger.jpa.jdbc.password": "_",
+ "ranger.spnego.kerberos.keytab": "",
+ "ranger.sso.providerurl": "",
+ "ranger.unixauth.service.hostname": "{{ugsync_host}}",
+ "ranger.admin.kerberos.keytab": "",
+ "ranger.admin.kerberos.token.valid.seconds": "30",
+ "ranger.jpa.jdbc.driver": "com.mysql.jdbc.Driver",
+ "ranger.unixauth.service.port": "5151"
+ },
+ "ranger-hdfs-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks",
+ "xasecure.policymgr.clientssl.truststore.password": "changeit",
+ "xasecure.policymgr.clientssl.keystore.credential.file": "jceks://file{{credential_file}}",
+ "xasecure.policymgr.clientssl.truststore": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks",
+ "xasecure.policymgr.clientssl.truststore.credential.file": "jceks://file{{credential_file}}",
+ "xasecure.policymgr.clientssl.keystore.password": "myKeyFilePassword"
+ },
+ "tagsync-application-properties": {
+ "atlas.kafka.entities.group.id": "ranger_entities_consumer",
+ "atlas.kafka.zookeeper.connect": "c6401.ambari.apache.org:2181",
+ "atlas.kafka.bootstrap.servers": "localhost:6667"
+ },
+ "ranger-env": {
+ "ranger_solr_shards": "1",
+ "ranger_solr_config_set": "ranger_audits",
+ "ranger_user": "ranger",
+ "ranger_solr_replication_factor": "1",
+ "xml_configurations_supported": "true",
+ "ranger-atlas-plugin-enabled": "No",
+ "ranger-hbase-plugin-enabled": "No",
+ "ranger-yarn-plugin-enabled": "No",
+ "bind_anonymous": "false",
+ "ranger_admin_username": "amb_ranger_admin",
+ "admin_password": "admin",
+ "is_solrCloud_enabled": "true",
+ "ranger-storm-plugin-enabled": "No",
+ "ranger-hdfs-plugin-enabled": "No",
+ "ranger_group": "ranger",
+ "ranger-knox-plugin-enabled": "No",
+ "ranger_admin_log_dir": "/var/log/ranger/admin",
+ "ranger-kafka-plugin-enabled": "No",
+ "ranger_privelege_user_jdbc_url": "jdbc:mysql://c6401.ambari.apache.org:3306",
+ "ranger-hive-plugin-enabled": "No",
+ "xasecure.audit.destination.solr": "true",
+ "ranger_pid_dir": "/var/run/ranger",
+ "xasecure.audit.destination.hdfs": "true",
+ "admin_username": "admin",
+ "xasecure.audit.destination.hdfs.dir": "hdfs://c6401.ambari.apache.org:8020/ranger/audit",
+ "create_db_dbuser": "true",
+ "ranger_solr_collection_name": "ranger_audits",
+ "ranger_admin_password": "P1!q9xa96SMi5NCl",
+ "ranger_usersync_log_dir": "/var/log/ranger/usersync"
+ },
+ "usersync-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nlog4j.rootLogger = info,logFile\n\n# logFile\nlog4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.logFile.file=${logdir}/usersync.
log\nlog4j.appender.logFile.datePattern='.'yyyy-MM-dd\nlog4j.appender.logFile.layout=org.apache.log4j.PatternLayout\nlog4j.appender.logFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n\n\n# console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.Target=System.out\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"
+ },
+ "admin-properties": {
+ "db_user": "rangeradmin01",
+ "DB_FLAVOR": "MYSQL",
+ "db_password": "rangeradmin01",
+ "db_root_user": "root",
+ "policymgr_external_url": "http://c6401.ambari.apache.org:6080",
+ "db_name": "ranger01",
+ "db_host": "c6401.ambari.apache.org",
+ "db_root_password": "vagrant",
+ "SQL_CONNECTOR_JAR": "{{driver_curl_target}}"
+ },
+ "ranger-ugsync-site": {
+ "ranger.usersync.ldap.binddn": "",
+ "ranger.usersync.policymgr.username": "rangerusersync",
+ "ranger.usersync.policymanager.mockrun": "false",
+ "ranger.usersync.group.searchbase": "",
+ "ranger.usersync.ldap.bindalias": "testldapalias",
+ "ranger.usersync.truststore.file": "/usr/hdp/current/ranger-usersync/conf/mytruststore.jks",
+ "ranger.usersync.port": "5151",
+ "ranger.usersync.pagedresultssize": "500",
+ "ranger.usersync.group.memberattributename": "",
+ "ranger.usersync.kerberos.principal": "",
+ "ranger.usersync.source.impl.class": "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder",
+ "ranger.usersync.ldap.referral": "ignore",
+ "ranger.usersync.group.searchfilter": "",
+ "ranger.usersync.ldap.user.objectclass": "person",
+ "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.ldap.user.searchfilter": "",
+ "ranger.usersync.ldap.groupname.caseconversion": "none",
+ "ranger.usersync.ldap.ldapbindpassword": "",
+ "ranger.usersync.unix.minUserId": "500",
+ "ranger.usersync.policymanager.maxrecordsperapicall": "1000",
+ "ranger.usersync.group.nameattribute": "",
+ "ranger.usersync.policymgr.alias": "ranger.usersync.policymgr.password",
+ "ranger.usersync.keystore.file": "/usr/hdp/current/ranger-usersync/conf/unixauthservice.jks",
+ "ranger.usersync.user.searchenabled": "false",
+ "ranger.usersync.group.usermapsyncenabled": "true",
+ "ranger.usersync.ldap.bindkeystore": "",
+ "ranger.usersync.ldap.user.groupnameattribute": "memberof, ismemberof",
+ "ranger.usersync.kerberos.keytab": "",
+ "ranger.usersync.passwordvalidator.path": "./native/credValidator.uexe",
+ "ranger.usersync.group.objectclass": "",
+ "ranger.usersync.ldap.user.searchscope": "sub",
+ "ranger.usersync.unix.password.file": "/etc/passwd",
+ "ranger.usersync.ldap.user.nameattribute": "",
+ "ranger.usersync.pagedresultsenabled": "true",
+ "ranger.usersync.policymanager.baseURL": "{{ranger_external_url}}",
+ "ranger.usersync.group.search.first.enabled": "false",
+ "ranger.usersync.group.searchenabled": "false",
+ "ranger.usersync.sink.impl.class": "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder",
+ "ranger.usersync.ssl": "true",
+ "ranger.usersync.ldap.url": "",
+ "ranger.usersync.ldap.searchBase": "dc=hadoop,dc=apache,dc=org",
+ "ranger.usersync.policymgr.keystore": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks",
+ "ranger.usersync.ldap.user.searchbase": "",
+ "ranger.usersync.ldap.username.caseconversion": "none",
+ "ranger.usersync.credstore.filename": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks",
+ "ranger.usersync.keystore.password": "UnIx529p",
+ "ranger.usersync.unix.group.file": "/etc/group",
+ "ranger.usersync.filesource.file": "/tmp/usergroup.txt",
+ "ranger.usersync.group.searchscope": "",
+ "ranger.usersync.truststore.password": "changeit",
+ "ranger.usersync.enabled": "true",
+ "ranger.usersync.sleeptimeinmillisbetweensynccycle": "60000",
+ "ranger.usersync.filesource.text.delimiter": ","
+ },
+ "hdfs-site": {
+ "dfs.namenode.checkpoint.period": "21600",
+ "dfs.namenode.avoid.write.stale.datanode": "true",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.namenode.checkpoint.txns": "1000000",
+ "dfs.content-summary.limit": "5000",
+ "dfs.support.append": "true",
+ "dfs.datanode.address": "0.0.0.0:50010",
+ "dfs.cluster.administrators": " hdfs",
+ "dfs.namenode.audit.log.async": "true",
+ "dfs.datanode.balance.bandwidthPerSec": "6250000",
+ "dfs.namenode.safemode.threshold-pct": "1",
+ "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
+ "dfs.namenode.rpc-address": "c6401.ambari.apache.org:8020",
+ "dfs.permissions.enabled": "true",
+ "dfs.client.read.shortcircuit": "true",
+ "dfs.https.port": "50470",
+ "dfs.namenode.https-address": "c6401.ambari.apache.org:50470",
+ "nfs.file.dump.dir": "/tmp/.hdfs-nfs",
+ "dfs.blocksize": "134217728",
+ "dfs.blockreport.initialDelay": "120",
+ "dfs.journalnode.edits.dir": "/hadoop/hdfs/journalnode",
+ "dfs.namenode.fslock.fair": "false",
+ "dfs.datanode.max.transfer.threads": "4096",
+ "dfs.heartbeat.interval": "3",
+ "dfs.replication": "3",
+ "dfs.namenode.handler.count": "50",
+ "dfs.namenode.checkpoint.dir": "/grid/0/hadoop/hdfs/namesecondary",
+ "fs.permissions.umask-mode": "022",
+ "dfs.namenode.stale.datanode.interval": "30000",
+ "dfs.datanode.ipc.address": "0.0.0.0:8010",
+ "dfs.datanode.failed.volumes.tolerated": "0",
+ "dfs.datanode.data.dir": "/grid/0/hadoop/hdfs/data",
+ "dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
+ "dfs.webhdfs.enabled": "true",
+ "dfs.encrypt.data.transfer.cipher.suites": "AES/CTR/NoPadding",
+ "dfs.namenode.accesstime.precision": "0",
+ "dfs.datanode.https.address": "0.0.0.0:50475",
+ "dfs.namenode.write.stale.datanode.ratio": "1.0f",
+ "dfs.namenode.secondary.http-address": "c6401.ambari.apache.org:50090",
+ "nfs.exports.allowed.hosts": "* rw",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
+ "dfs.datanode.du.reserved": "33011188224",
+ "dfs.client.read.shortcircuit.streams.cache.size": "4096",
+ "dfs.http.policy": "HTTP_ONLY",
+ "dfs.block.access.token.enable": "true",
+ "dfs.client.retry.policy.enabled": "false",
+ "dfs.namenode.name.dir.restore": "true",
+ "dfs.permissions.superusergroup": "hdfs",
+ "dfs.journalnode.https-address": "0.0.0.0:8481",
+ "dfs.journalnode.http-address": "0.0.0.0:8480",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "dfs.namenode.avoid.read.stale.datanode": "true",
+ "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude",
+ "dfs.datanode.data.dir.perm": "750",
+ "dfs.encryption.key.provider.uri": "",
+ "dfs.replication.max": "50",
+ "dfs.namenode.name.dir": "/grid/0/hadoop/hdfs/namenode"
+ },
+ "ranger-tagsync-site": {
+ "ranger.tagsync.dest.ranger.ssl.config.filename": "/usr/hdp/current/ranger-tagsync/conf/mytruststore.jks",
+ "ranger.tagsync.source.atlasrest.username": "",
+ "ranger.tagsync.logdir": "/var/log/ranger/tagsync",
+ "ranger.tagsync.source.atlasrest.download.interval.millis": "",
+ "ranger.tagsync.keystore.filename": "/usr/hdp/current/ranger-tagsync/conf/rangertagsync.jceks",
+ "ranger.tagsync.source.file.check.interval.millis": "",
+ "ranger.tagsync.source.atlasrest.endpoint": "",
+ "ranger.tagsync.dest.ranger.username": "rangertagsync",
+ "ranger.tagsync.dest.ranger.endpoint": "{{ranger_external_url}}",
+ "ranger.tagsync.kerberos.principal": "",
+ "ranger.tagsync.kerberos.keytab": "",
+ "ranger.tagsync.source.atlas": "false",
+ "ranger.tagsync.source.atlasrest": "false",
+ "ranger.tagsync.source.file": "false",
+ "ranger.tagsync.source.file.filename": ""
+ },
+ "zoo.cfg": {
+ "clientPort": "2181",
+ "autopurge.purgeInterval": "24",
+ "syncLimit": "5",
+ "dataDir": "/grid/0/hadoop/zookeeper",
+ "initLimit": "10",
+ "tickTime": "2000",
+ "autopurge.snapRetainCount": "30"
+ },
+ "hadoop-policy": {
+ "security.job.client.protocol.acl": "*",
+ "security.job.task.protocol.acl": "*",
+ "security.datanode.protocol.acl": "*",
+ "security.namenode.protocol.acl": "*",
+ "security.client.datanode.protocol.acl": "*",
+ "security.inter.tracker.protocol.acl": "*",
+ "security.refresh.usertogroups.mappings.protocol.acl": "hadoop",
+ "security.client.protocol.acl": "*",
+ "security.refresh.policy.protocol.acl": "hadoop",
+ "security.admin.operations.protocol.acl": "hadoop",
+ "security.inter.datanode.protocol.acl": "*"
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.lo
gger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender
.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.security.logg
er}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=
false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# NameNode metrics logging.\n# The default is to retain two namenode-metrics.log files up to 64MB each.\n#\nnamenode.metrics.logger=INFO,NullAppender\nlog4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}\nlog4j.additivity.NameNodeMetricsLog=false\nlog4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log\nlog4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n\nlog4j.appender.NNMETRICSRFA.MaxBackupIndex=1\nlog4j.appender.NNMETRICSRFA.MaxFileSize=64MB\n\n#\n# mapred audit logging\n#\nmapred
.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging leve
ls\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateCh
ange=WARN"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "hadoop.rpc.protection": "authentication",
+ "ranger-hdfs-plugin-enabled": "No",
+ "REPOSITORY_CONFIG_USERNAME": "hadoop",
+ "policy_user": "ambari-qa",
+ "common.name.for.certificate": "",
+ "REPOSITORY_CONFIG_PASSWORD": "hadoop"
+ },
+ "core-site": {
+ "hadoop.proxyuser.root.hosts": "c6401.ambari.apache.org",
+ "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
+ "fs.trash.interval": "360",
+ "ha.failover-controller.active-standby-elector.zk.op.retries": "120",
+ "hadoop.http.authentication.simple.anonymous.allowed": "true",
+ "hadoop.security.authentication": "simple",
+ "hadoop.proxyuser.root.groups": "*",
+ "ipc.client.connection.maxidletime": "30000",
+ "hadoop.security.key.provider.path": "",
+ "mapreduce.jobtracker.webinterface.trusted": "false",
+ "hadoop.security.authorization": "false",
+ "net.topology.script.file.name": "/etc/hadoop/conf/topology_script.py",
+ "ipc.server.tcpnodelay": "true",
+ "ipc.client.connect.max.retries": "50",
+ "hadoop.security.auth_to_local": "DEFAULT",
+ "io.file.buffer.size": "131072",
+ "hadoop.proxyuser.hdfs.hosts": "*",
+ "hadoop.proxyuser.hdfs.groups": "*",
+ "ipc.client.idlethreshold": "8000",
+ "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020",
+ "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec"
+ },
+ "hadoop-env": {
+ "keyserver_port": "",
+ "proxyuser_group": "users",
+ "hdfs_user_nproc_limit": "65536",
+ "hdfs_log_dir_prefix": "/var/log/hadoop",
+ "hdfs_user_nofile_limit": "128000",
+ "hdfs_user": "hdfs",
+ "keyserver_host": " ",
+ "namenode_opt_maxnewsize": "128m",
+ "namenode_opt_maxpermsize": "256m",
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appende
d to HADOOP_OPTS when specified\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\n\n{% if java_version < 8 %}\nSHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSiz
e={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\nexport HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms
{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n{% else %}\nSHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{name
node_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\nexport HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARY
NAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n{% endif %}\n\nHADOOP_NFS3_OPTS=\"-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_D
N_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The schedul
ing priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Add database libraries\nJAVA_JDBC_LIBS=\"\"\nif [ -d \"/usr/share/java\" ]; then\n for jarFile in `ls /usr/share/java | grep -E \"(mysql|ojdbc|postgresql|sqljdbc)\" 2>/dev/null`\n do\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\n done\nfi\n\n# Add libraries to the hadoop classpath - some may not need a colon as they already include it\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\n\n{% if is_datanode_max_locked_memory_set %}\n# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. \n# Makes sense to fix only when runing DN as root \nif [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_S
ECURE_DN_USER\" ]; then\n ulimit -l {{datanode_max_locked_memory}}\nfi\n{% endif %}",
+ "namenode_heapsize": "1024m",
+ "namenode_opt_newsize": "128m",
+ "nfsgateway_heapsize": "1024",
+ "dtnode_heapsize": "1024m",
+ "hadoop_root_logger": "INFO,RFA",
+ "hadoop_heapsize": "1024",
+ "hadoop_pid_dir_prefix": "/var/run/hadoop",
+ "namenode_opt_permsize": "128m",
+ "hdfs_tmp_dir": "/tmp"
+ },
+ "zookeeper-log4j": {
+ "content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4
j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN
GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
+ },
+ "ssl-server": {
+ "ssl.server.keystore.location": "/etc/security/serverKeys/keystore.jks",
+ "ssl.server.keystore.keypassword": "bigdata",
+ "ssl.server.truststore.location": "/etc/security/serverKeys/all.jks",
+ "ssl.server.keystore.password": "bigdata",
+ "ssl.server.truststore.password": "bigdata",
+ "ssl.server.truststore.type": "jks",
+ "ssl.server.keystore.type": "jks",
+ "ssl.server.truststore.reload.interval": "10000"
+ },
+ "ranger-site": {},
+ "admin-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nlog4j.rootLogger = warn,xa_log_appender\n\n\n# xa_logger\nlog4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.xa_log_app
ender.file=${logdir}/xa_portal.log\nlog4j.appender.xa_log_appender.datePattern='.'yyyy-MM-dd\nlog4j.appender.xa_log_appender.append=true\nlog4j.appender.xa_log_appender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.xa_log_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n\n# xa_log_appender : category and additivity\nlog4j.category.org.springframework=warn,xa_log_appender\nlog4j.additivity.org.springframework=false\n\nlog4j.category.org.apache.ranger=info,xa_log_appender\nlog4j.additivity.org.apache.ranger=false\n\nlog4j.category.xa=info,xa_log_appender\nlog4j.additivity.xa=false\n\n# perf_logger\nlog4j.appender.perf_appender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.perf_appender.file=${logdir}/ranger_admin_perf.log\nlog4j.appender.perf_appender.datePattern='.'yyyy-MM-dd\nlog4j.appender.perf_appender.append=true\nlog4j.appender.perf_appender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.perf_appender.layout.ConversionPattern=%d [%t]
%m%n\n\n\n# sql_appender\nlog4j.appender.sql_appender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.sql_appender.file=${logdir}/xa_portal_sql.log\nlog4j.appender.sql_appender.datePattern='.'yyyy-MM-dd\nlog4j.appender.sql_appender.append=true\nlog4j.appender.sql_appender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.sql_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n\n\n# sql_appender : category and additivity\nlog4j.category.org.hibernate.SQL=warn,sql_appender\nlog4j.additivity.org.hibernate.SQL=false\n\nlog4j.category.jdbc.sqlonly=fatal,sql_appender\nlog4j.additivity.jdbc.sqlonly=false\n\nlog4j.category.jdbc.sqltiming=warn,sql_appender\nlog4j.additivity.jdbc.sqltiming=false\n\nlog4j.category.jdbc.audit=fatal,sql_appender\nlog4j.additivity.jdbc.audit=false\n\nlog4j.category.jdbc.resultset=fatal,sql_appender\nlog4j.additivity.jdbc.resultset=false\n\nlog4j.category.jdbc.connection=fatal,sql_appender\nlog4j.additivity.jdbc.connection=false"
+ },
+ "tagsync-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nlog4j.rootLogger = info,logFile\n\n# logFile\nlog4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.logFile.file=${logdir}/tagsync
.log\nlog4j.appender.logFile.datePattern='.'yyyy-MM-dd\nlog4j.appender.logFile.layout=org.apache.log4j.PatternLayout\nlog4j.appender.logFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n\n\n# console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.Target=System.out\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"
+ },
+ "ranger-hdfs-security": {
+ "ranger.plugin.hdfs.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "ranger.plugin.hdfs.service.name": "{{repo_name}}",
+ "ranger.plugin.hdfs.policy.cache.dir": "/etc/ranger/{{repo_name}}/policycache",
+ "ranger.plugin.hdfs.policy.pollIntervalMs": "30000",
+ "ranger.plugin.hdfs.policy.rest.url": "{{policymgr_mgr_url}}",
+ "ranger.plugin.hdfs.policy.rest.ssl.config.file": "/etc/hadoop/conf/ranger-policymgr-ssl.xml",
+ "xasecure.add-hadoop-authorization": "true"
+ },
+ "usersync-properties": {},
+ "zookeeper-env": {
+ "zk_log_dir": "/var/log/zookeeper",
+ "content": "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}",
+ "zk_server_heapsize": "1024m",
+ "zk_pid_dir": "/var/run/zookeeper",
+ "zk_user": "zookeeper"
+ },
+ "infra-solr-env": {
+ "infra_solr_datadir": "/opt/logsearch_solr/data",
+ "infra_solr_keystore_location": "/etc/security/serverKeys/logsearch.keyStore.jks",
+ "infra_solr_kerberos_name_rules": "DEFAULT",
+ "infra_solr_user": "infra-solr",
+ "infra_solr_maxmem": "1024",
+ "content": "#!/bin/bash\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# By default the script will use JAVA_HOME to determine which java\n# to use, but you can set a specific path for Solr to use without\n# affecting other Java applica
tions on your server/workstation.\nSOLR_JAVA_HOME={{java64_home}}\n\n# Increase Java Min/Max Heap as needed to support your indexing / query needs\nSOLR_JAVA_MEM=\"-Xms{{logsearch_solr_min_mem}}m -Xmx{{logsearch_solr_max_mem}}m\"\n\n# Enable verbose GC logging\nGC_LOG_OPTS=\"-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \\\n-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime\"\n\n# These GC settings have shown to work well for a number of common Solr workloads\nGC_TUNE=\"-XX:NewRatio=3 \\\n-XX:SurvivorRatio=4 \\\n-XX:TargetSurvivorRatio=90 \\\n-XX:MaxTenuringThreshold=8 \\\n-XX:+UseConcMarkSweepGC \\\n-XX:+UseParNewGC \\\n-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \\\n-XX:+CMSScavengeBeforeRemark \\\n-XX:PretenureSizeThreshold=64m \\\n-XX:+UseCMSInitiatingOccupancyOnly \\\n-XX:CMSInitiatingOccupancyFraction=50 \\\n-XX:CMSMaxAbortablePrecleanTime=6000 \\\n-XX:+CMSParallelRemarkEnabled \\\n-XX:+ParallelRefProcEnabled\"\
n\n# Set the ZooKeeper connection string if using an external ZooKeeper ensemble\n# e.g. host1:2181,host2:2181/chroot\n# Leave empty if not using SolrCloud\nZK_HOST=\"{{zookeeper_quorum}}{{logsearch_solr_znode}}\"\n\n# Set the ZooKeeper client timeout (for SolrCloud mode)\nZK_CLIENT_TIMEOUT=\"60000\"\n\n# By default the start script uses \"localhost\"; override the hostname here\n# for production SolrCloud environments to control the hostname exposed to cluster state\n#SOLR_HOST=\"192.168.1.1\"\n\n# By default the start script uses UTC; override the timezone if needed\n#SOLR_TIMEZONE=\"UTC\"\n\n# Set to true to activate the JMX RMI connector to allow remote JMX client applications\n# to monitor the JVM hosting Solr; set to \"false\" to disable that behavior\n# (false is recommended in production environments)\nENABLE_REMOTE_JMX_OPTS=\"true\"\n\n# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here\nRMI_PORT={{logsearch_solr_jmx_port}}\n\n# Anything you add to
the SOLR_OPTS variable will be included in the java\n# start command line as-is, in ADDITION to other options. If you specify the\n# -a option on start script, those options will be appended as well. Examples:\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.clustering.enabled=true\"\n\n# Location where the bin/solr script will save PID files for running instances\n# If not set, the script will create PID files in $SOLR_TIP/bin\nSOLR_PID_DIR={{logsearch_solr_piddir}}\n\n# Path to a directory where Solr creates index files, the specified directory\n# must contain a solr.xml; by default, Solr will use server/solr\nSOLR_HOME={{logsearch_solr_datadir}}\n\n# Solr provides a default Log4J configuration properties file in server/resources\n# however, you may want to customize the log settings and file appender location\n# so you can point the script to use a different log4j.properties f
ile\nLOG4J_PROPS={{logsearch_solr_conf}}/log4j.properties\n\n# Location where Solr should write logs to; should agree with the file appender\n# settings in server/resources/log4j.properties\nSOLR_LOGS_DIR={{logsearch_solr_log_dir}}\n\n# Sets the port Solr binds to, default is 8983\nSOLR_PORT={{logsearch_solr_port}}\n\n# Be sure to update the paths to the correct keystore for your environment\n{% if logsearch_solr_ssl_enabled %}\nSOLR_SSL_KEY_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_KEY_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_TRUST_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_TRUST_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_NEED_CLIENT_AUTH=false\nSOLR_SSL_WANT_CLIENT_AUTH=false\n{% endif %}\n\n# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set\n# and you are using SSL, then the start script will use SOLR_PORT for the SSL port\n#SOLR_SSL_PORT=\n\n{% if security_enabled -%}\nSOLR_HOST=`hostname -f`\nSOLR
_JAAS_FILE={{logsearch_solr_jaas_file}}\nSOLR_KERB_KEYTAB={{logsearch_solr_web_kerberos_keytab}}\nSOLR_KERB_PRINCIPAL={{logsearch_solr_web_kerberos_principal}}\nSOLR_KERB_NAME_RULES={{logsearch_solr_kerberos_name_rules}}\n\nSOLR_AUTHENTICATION_CLIENT_CONFIGURER=\"org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer\"\nSOLR_AUTHENTICATION_OPTS=\" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=$SOLR_JAAS_FILE -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST} -Dsolr.kerberos.name.rules=${SOLR_KERB_NAME_RULES}\"\n{% endif %}",
+ "infra_solr_pid_dir": "/var/run/ambari-infra-solr",
+ "infra_solr_truststore_password": "bigdata",
+ "infra_solr_truststore_type": "jks",
+ "infra_solr_keystore_type": "jks",
+ "infra_solr_log_dir": "/var/log/ambari-infra-solr",
+ "infra_solr_web_kerberos_keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "infra_solr_ssl_enabled": "false",
+ "infra_solr_client_log_dir": "/var/log/ambari-infra-solr-client",
+ "infra_solr_web_kerberos_principal": "HTTP/_HOST@EXAMPLE.COM",
+ "infra_solr_znode": "/infra-solr",
+ "infra_solr_keystore_password": "bigdata",
+ "infra_solr_port": "8886",
+ "infra_solr_kerberos_principal": "infra-solr/_HOST@EXAMPLE.COM",
+ "infra_solr_jmx_port": "18886",
+ "infra_solr_truststore_location": "/etc/security/serverKeys/logsearch.trustStore.jks",
+ "infra_solr_minmem": "512",
+ "infra_solr_kerberos_keytab": "/etc/security/keytabs/infra-solr.service.keytab"
+ },
+ "infra-solr-client-log4j": {
+ "infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
+ "content": "# Copyright 2011 The Apache Software Foundation\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlog4j.rootLogger=INFO,file,stdout,stderr\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender
.file.File={{logsearch_solr_client_log}}\nlog4j.appender.file.MaxFileSize=80MB\nlog4j.appender.file.MaxBackupIndex=60\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.Threshold=INFO\nlog4j.appender.stdout.Target=System.out\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=%m%n\n\nlog4j.appender.stderr=org.apache.log4j.ConsoleAppender\nlog4j.appender.stderr.Threshold=ERROR\nlog4j.appender.stderr.Target=System.err\nlog4j.appender.stderr.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stderr.layout.ConversionPattern=%m%n"
+ },
+ "cluster-env": {
+ "security_enabled": "false",
+ "override_uid": "true",
+ "fetch_nonlocal_groups": "true",
+ "one_dir_per_partition": "true",
+ "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
+ "ignore_groupsusers_create": "false",
+ "alerts_repeat_tolerance": "1",
+ "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
+ "kerberos_domain": "EXAMPLE.COM",
+ "manage_dirs_on_root": "true",
+ "recovery_lifetime_max_count": "1024",
+ "recovery_type": "AUTO_START",
+ "ignore_bad_mounts": "false",
+ "recovery_window_in_minutes": "60",
+ "user_group": "hadoop",
+ "stack_tools": "{\n \"stack_selector\": [\"hdp-select\", \"/usr/bin/hdp-select\", \"hdp-select\"],\n \"conf_selector\": [\"conf-select\", \"/usr/bin/conf-select\", \"conf-select\"]\n}",
+ "recovery_retry_interval": "5",
+ "stack_features": "{\n \"stack_features\": [\n {\n \"name\": \"snappy\",\n \"description\": \"Snappy compressor/decompressor support\",\n \"min_version\": \"2.0.0.0\",\n \"max_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"lzo\",\n \"description\": \"LZO libraries support\",\n \"min_version\": \"2.2.1.0\"\n },\n {\n \"name\": \"express_upgrade\",\n \"description\": \"Express upgrade support\",\n \"min_version\": \"2.1.0.0\"\n },\n {\n \"name\": \"rolling_upgrade\",\n \"description\": \"Rolling upgrade support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"config_versioning\",\n \"description\": \"Configurable versions support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"datanode_non_root\",\n \"description\": \"DataNode running as non-root support (AMBARI-7615)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\
": \"remove_ranger_hdfs_plugin_env\",\n \"description\": \"HDFS removes Ranger env files (AMBARI-14299)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger\",\n \"description\": \"Ranger Service support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_tagsync_component\",\n \"description\": \"Ranger Tagsync component support (AMBARI-14383)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"phoenix\",\n \"description\": \"Phoenix Service support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"nfs\",\n \"description\": \"NFS support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"tez_for_spark\",\n \"description\": \"Tez dependency for Spark\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"timeline_state_store\",\n \"description\": \"Yarn application timeline-se
rvice supports state store property (AMBARI-11442)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"copy_tarball_to_hdfs\",\n \"description\": \"Copy tarball to HDFS support (AMBARI-12113)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"spark_16plus\",\n \"description\": \"Spark 1.6+\",\n \"min_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"spark_thriftserver\",\n \"description\": \"Spark Thrift Server\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"storm_kerberos\",\n \"description\": \"Storm Kerberos support (AMBARI-7570)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"storm_ams\",\n \"description\": \"Storm AMS integration (AMBARI-10710)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"create_kafka_broker_id\",\n \"description\": \"Ambari should create Kafka Broker Id (AMBARI-12678)\",\n \"min_version\": \"2.2
.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_listeners\",\n \"description\": \"Kafka listeners (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_kerberos\",\n \"description\": \"Kafka Kerberos support (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"pig_on_tez\",\n \"description\": \"Pig on Tez support (AMBARI-7863)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_usersync_non_root\",\n \"description\": \"Ranger Usersync as non-root user (AMBARI-10416)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger_audit_db_support\",\n \"description\": \"Ranger Audit to DB support\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"accumulo_kerberos_user_auth\",\n \"description\": \"Accumulo Kerberos User Auth (AMBARI-10163)\",\n
\"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"knox_versioned_data_dir\",\n \"description\": \"Use versioned data dir for Knox (AMBARI-13164)\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"knox_sso_topology\",\n \"description\": \"Knox SSO Topology support (AMBARI-13975)\",\n \"min_version\": \"2.3.8.0\"\n },\n {\n \"name\": \"atlas_rolling_upgrade\",\n \"description\": \"Rolling upgrade support for Atlas\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"oozie_admin_user\",\n \"description\": \"Oozie install user as an Oozie admin user (AMBARI-7976)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_create_hive_tez_configs\",\n \"description\": \"Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_setup_shared_lib\",\n \"description\": \"Oozie
setup tools used to shared Oozie lib to HDFS (AMBARI-7240)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_host_kerberos\",\n \"description\": \"Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)\",\n \"min_version\": \"2.0.0.0\",\n \"max_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"falcon_extensions\",\n \"description\": \"Falcon Extension\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_upgrade_schema\",\n \"description\": \"Hive metastore upgrade schema support (AMBARI-11176)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server_interactive\",\n \"description\": \"Hive server interactive support (AMBARI-15573)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_webhcat_specific_configs\",\n \"description\": \"Hive webhcat specific configurations support (AMBARI-12364)\",\n \"min_ver
sion\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_purge_table\",\n \"description\": \"Hive purge table support (AMBARI-12260)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server2_kerberized_env\",\n \"description\": \"Hive server2 working on kerberized environment (AMBARI-13749)\",\n \"min_version\": \"2.2.3.0\",\n \"max_version\": \"2.2.5.0\"\n },\n {\n \"name\": \"hive_env_heapsize\",\n \"description\": \"Hive heapsize property defined in hive-env (AMBARI-12801)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_kms_hsm_support\",\n \"description\": \"Ranger KMS HSM support (AMBARI-15752)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_log4j_support\",\n \"description\": \"Ranger supporting log-4j properties (AMBARI-15681)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kerberos_support\",\n \"desc
ription\": \"Ranger Kerberos support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_site_support\",\n \"description\": \"Hive Metastore site support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_usersync_password_jceks\",\n \"description\": \"Saving Ranger Usersync credentials in jceks\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_install_infra_client\",\n \"description\": \"LogSearch Service support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hbase_home_directory\",\n \"description\": \"Hbase home directory in HDFS needed for HBASE backup\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_livy\",\n \"description\": \"Livy as slave component of spark\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"atlas_ranger_plugin_support\",\n \"description\": \"Atlas Ranger plug
in support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_pid_support\",\n \"description\": \"Ranger Service support pid generation AMBARI-16756\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kms_pid_support\",\n \"description\": \"Ranger KMS Service support pid generation\",\n \"min_version\": \"2.5.0.0\"\n }\n ]\n}",
+ "recovery_enabled": "true",
+ "recovery_max_count": "6",
+ "stack_root": "/usr/hdp",
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "managed_hdfs_resource_property_names": "",
+ "smokeuser": "ambari-qa"
+ }
+ }
+}
\ No newline at end of file
[47/50] [abbrv] ambari git commit: AMBARI-19089. Manage Journalnode
Wizard is stuck at Add/Remove JournalNodes in Kerberized Cluster (akovalenko)
Posted by nc...@apache.org.
AMBARI-19089. Manage Journalnode Wizard is stuck at Add/Remove JournalNodes in Kerberized Cluster (akovalenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49d08958
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49d08958
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49d08958
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 49d089586381f44882c57e0a5b8e029e775fe1de
Parents: 165c4d6
Author: Aleksandr Kovalenko <ak...@hortonworks.com>
Authored: Wed Dec 7 12:54:02 2016 +0200
Committer: Aleksandr Kovalenko <ak...@hortonworks.com>
Committed: Wed Dec 7 13:01:58 2016 +0200
----------------------------------------------------------------------
.../admin/highAvailability/journalNode/step4_controller.js | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/49d08958/ambari-web/app/controllers/main/admin/highAvailability/journalNode/step4_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/admin/highAvailability/journalNode/step4_controller.js b/ambari-web/app/controllers/main/admin/highAvailability/journalNode/step4_controller.js
index 329b8fe..1938527 100644
--- a/ambari-web/app/controllers/main/admin/highAvailability/journalNode/step4_controller.js
+++ b/ambari-web/app/controllers/main/admin/highAvailability/journalNode/step4_controller.js
@@ -60,11 +60,7 @@ App.ManageJournalNodeWizardStep4Controller = App.ManageJournalNodeProgressPageCo
reconfigureHDFS: function () {
var data = this.get('content.serviceConfigProperties');
- if (App.get('isKerberosEnabled')) {
- // TODO this.reconfigureSecureHDFS();
- } else {
- this.updateConfigProperties(data);
- }
+ this.updateConfigProperties(data);
},
/**
[17/50] [abbrv] ambari git commit: AMBARI-19084. Custom property
added to the Notification settings appears twice (onechiporenko)
Posted by nc...@apache.org.
AMBARI-19084. Custom property added to the Notification settings appears twice (onechiporenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b2d2ef16
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b2d2ef16
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b2d2ef16
Branch: refs/heads/branch-dev-patch-upgrade
Commit: b2d2ef1689d5434c20d6a791ace598f58742605a
Parents: f5051dc
Author: Oleg Nechiporenko <on...@apache.org>
Authored: Mon Dec 5 17:08:41 2016 +0200
Committer: Oleg Nechiporenko <on...@apache.org>
Committed: Mon Dec 5 18:05:42 2016 +0200
----------------------------------------------------------------------
.../common/configs/notifications_configs.hbs | 2 +-
.../notification_configs_view.js | 18 ++++++++++++++----
.../configs/service_configs_by_category_view.js | 9 ++++++++-
3 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b2d2ef16/ambari-web/app/templates/common/configs/notifications_configs.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/common/configs/notifications_configs.hbs b/ambari-web/app/templates/common/configs/notifications_configs.hbs
index ca2c208..2dd71aa 100644
--- a/ambari-web/app/templates/common/configs/notifications_configs.hbs
+++ b/ambari-web/app/templates/common/configs/notifications_configs.hbs
@@ -20,7 +20,7 @@
<div class="panel-heading" {{action "onToggleBlock" category target="view"}}>
<h3 class="panel-title">
<i {{bindAttr class=":pull-left :panel-toggle view.category.isCollapsed:icon-caret-right:icon-caret-down"}}></i>
- <a class="panel-toggle">{{view.category.displayName}}</a>
+ <a class="panel-toggle category-header"><span class="category-name">{{view.category.displayName}}</span></a>
</h3>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b2d2ef16/ambari-web/app/views/common/configs/custom_category_views/notification_configs_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/configs/custom_category_views/notification_configs_view.js b/ambari-web/app/views/common/configs/custom_category_views/notification_configs_view.js
index 9ad8762..d299b5f 100644
--- a/ambari-web/app/views/common/configs/custom_category_views/notification_configs_view.js
+++ b/ambari-web/app/views/common/configs/custom_category_views/notification_configs_view.js
@@ -62,8 +62,8 @@ App.NotificationsConfigsView = App.ServiceConfigsByCategoryView.extend({
if (!this.get('categoryConfigsAll.length')) return;
this.set('createNotification', this.get('categoryConfigsAll').findProperty('name', 'create_notification').get('value'));
this.set('tlsOrSsl', this.get('categoryConfigsAll').findProperty('name', 'mail.smtp.starttls.enable').get('value') ? 'tls' : 'ssl');
- var smtp_use_auth = this.get('categoryConfigsAll').findProperty('name', 'smtp_use_auth');
- smtp_use_auth.set('value', Boolean(smtp_use_auth.get('value') === 'true'));
+ var smtpUseAuth = this.get('categoryConfigsAll').findProperty('name', 'smtp_use_auth');
+ smtpUseAuth.set('value', Boolean(smtpUseAuth.get('value') === 'true'));
this.updateCategoryConfigs();
},
@@ -74,8 +74,8 @@ App.NotificationsConfigsView = App.ServiceConfigsByCategoryView.extend({
*/
onTlsOrSslChanged: function () {
var tlsOrSsl = this.get('tlsOrSsl');
- this.get('categoryConfigsAll').findProperty('name', 'mail.smtp.starttls.enable').set('value', tlsOrSsl == 'tls');
- this.get('categoryConfigsAll').findProperty('name', 'mail.smtp.startssl.enable').set('value', tlsOrSsl == 'ssl');
+ this.get('categoryConfigsAll').findProperty('name', 'mail.smtp.starttls.enable').set('value', tlsOrSsl === 'tls');
+ this.get('categoryConfigsAll').findProperty('name', 'mail.smtp.startssl.enable').set('value', tlsOrSsl === 'ssl');
}.observes('tlsOrSsl'),
/**
@@ -122,6 +122,16 @@ App.NotificationsConfigsView = App.ServiceConfigsByCategoryView.extend({
updateConfig: function (config, flag) {
config.set('isRequired', flag);
config.set('isEditable', flag);
+ },
+
+ /**
+ * No sense to store config to <code>serviceConfigs</code> and <code>categoryConfigsAll</code> because
+ * <code>categoryConfigsAll</code> is a subset of <code>serviceConfigs</code>
+ *
+ * @override
+ */
+ _appendConfigToCollection: function (serviceConfigProperty) {
+ this.get('serviceConfigs').pushObject(serviceConfigProperty);
}
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/b2d2ef16/ambari-web/app/views/common/configs/service_configs_by_category_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/configs/service_configs_by_category_view.js b/ambari-web/app/views/common/configs/service_configs_by_category_view.js
index dfcd5df..16258cf 100644
--- a/ambari-web/app/views/common/configs/service_configs_by_category_view.js
+++ b/ambari-web/app/views/common/configs/service_configs_by_category_view.js
@@ -439,7 +439,14 @@ App.ServiceConfigsByCategoryView = Em.View.extend(App.UserPref, App.ConfigOverri
isNotSaved: true
}, selectedConfigGroup);
}
- var serviceConfigProperty = App.ServiceConfigProperty.create(config);
+ this._appendConfigToCollection(App.ServiceConfigProperty.create(config));
+ },
+
+ /**
+ * @param {App.ServiceConfigProperty} serviceConfigProperty
+ * @private
+ */
+ _appendConfigToCollection: function (serviceConfigProperty) {
this.get('serviceConfigs').pushObject(serviceConfigProperty);
this.get('categoryConfigsAll').pushObject(serviceConfigProperty);
},
[07/50] [abbrv] ambari git commit: AMBARI-19055 - Removing Tasks From
host_role_command Causes Upgrades To Show As PENDING (jonathanhurley)
Posted by nc...@apache.org.
AMBARI-19055 - Removing Tasks From host_role_command Causes Upgrades To Show As PENDING (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/12a3cd87
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/12a3cd87
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/12a3cd87
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 12a3cd87184167cadf3cd12eb9b3c9d67dd3907a
Parents: 5ec73bc
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Sat Dec 3 08:35:12 2016 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Sat Dec 3 14:09:44 2016 -0500
----------------------------------------------------------------------
.../controller/internal/CalculatedStatus.java | 52 +++++++++++++++-----
.../internal/RequestResourceProvider.java | 34 ++++++++-----
.../internal/StageResourceProvider.java | 2 +-
.../ambari/server/topology/TopologyManager.java | 19 ++++++-
.../internal/CalculatedStatusTest.java | 31 ++++++++++++
.../internal/RequestResourceProviderTest.java | 12 +++--
6 files changed, 121 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/12a3cd87/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
index 3a86aef..3c415df 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CalculatedStatus.java
@@ -53,6 +53,17 @@ public class CalculatedStatus {
*/
private final double percent;
+ /**
+ * A status which represents a COMPLETED state at 0%
+ */
+ public static final CalculatedStatus COMPLETED = new CalculatedStatus(HostRoleStatus.COMPLETED,
+ HostRoleStatus.COMPLETED, 100.0);
+
+ /**
+ * A status which represents a PENDING state at 0%
+ */
+ public static final CalculatedStatus PENDING = new CalculatedStatus(HostRoleStatus.PENDING,
+ HostRoleStatus.PENDING, 0.0);
// ----- Constructors ------------------------------------------------------
@@ -79,12 +90,6 @@ public class CalculatedStatus {
this.percent = percent;
}
- /**
- * Static factory method to get Status that represents a Completed state
- */
- public static CalculatedStatus getCompletedStatus() {
- return new CalculatedStatus(HostRoleStatus.COMPLETED, HostRoleStatus.COMPLETED, 100.0);
- }
// ----- CalculatedStatus --------------------------------------------------
@@ -291,14 +296,25 @@ public class CalculatedStatus {
}
/**
- * Calculates the overall status of an upgrade.
- * @param stageDto the map of stage-to-summary value objects
- * @param stageIds the stage ids to consider from the value objects
+ * Calculates the overall status of an upgrade. If there are no tasks, then a
+ * status of {@link HostRoleStatus#COMPLETED} is returned.
+ *
+ * @param stageDto
+ * the map of stage-to-summary value objects
+ * @param stageIds
+ * the stage ids to consider from the value objects
* @return the calculated status
*/
public static CalculatedStatus statusFromStageSummary(Map<Long, HostRoleCommandStatusSummaryDTO> stageDto,
Set<Long> stageIds) {
+ // if either are empty, then we have no tasks and therefore no status - we
+ // should return COMPLETED. This can happen if someone removes all tasks but
+ // leaves the stages and request
+ if (stageDto.isEmpty() || stageIds.isEmpty()) {
+ return COMPLETED;
+ }
+
Collection<HostRoleStatus> stageStatuses = new HashSet<>();
Collection<HostRoleStatus> stageDisplayStatuses = new HashSet<>();
Collection<HostRoleStatus> taskStatuses = new ArrayList<>();
@@ -378,19 +394,28 @@ public class CalculatedStatus {
*/
public static HostRoleStatus calculateSummaryStatusOfStage(Map<HostRoleStatus, Integer> counters,
int total, boolean skippable) {
+
+ // when there are 0 tasks, return COMPLETED
+ if (total == 0) {
+ return HostRoleStatus.COMPLETED;
+ }
+
if (counters.get(HostRoleStatus.PENDING) == total) {
return HostRoleStatus.PENDING;
}
+
// By definition, any tasks in a future stage must be held in a PENDING status.
if (counters.get(HostRoleStatus.HOLDING) > 0 || counters.get(HostRoleStatus.HOLDING_FAILED) > 0 || counters.get(HostRoleStatus.HOLDING_TIMEDOUT) > 0) {
return counters.get(HostRoleStatus.HOLDING) > 0 ? HostRoleStatus.HOLDING :
counters.get(HostRoleStatus.HOLDING_FAILED) > 0 ? HostRoleStatus.HOLDING_FAILED :
HostRoleStatus.HOLDING_TIMEDOUT;
}
+
// Because tasks are not skippable, guaranteed to be FAILED
if (counters.get(HostRoleStatus.FAILED) > 0 && !skippable) {
return HostRoleStatus.FAILED;
}
+
// Because tasks are not skippable, guaranteed to be TIMEDOUT
if (counters.get(HostRoleStatus.TIMEDOUT) > 0 && !skippable) {
return HostRoleStatus.TIMEDOUT;
@@ -401,9 +426,11 @@ public class CalculatedStatus {
if (counters.get(HostRoleStatus.ABORTED) > 0 && numActiveTasks == 0) {
return HostRoleStatus.ABORTED;
}
+
if (counters.get(HostRoleStatus.COMPLETED) == total) {
return HostRoleStatus.COMPLETED;
}
+
return HostRoleStatus.IN_PROGRESS;
}
@@ -415,7 +442,8 @@ public class CalculatedStatus {
*
* @return summary request status based on statuses of tasks in different states.
*/
- private static HostRoleStatus calculateSummaryStatusOfUpgrade(Map<HostRoleStatus, Integer> counters, int total) {
+ protected static HostRoleStatus calculateSummaryStatusOfUpgrade(
+ Map<HostRoleStatus, Integer> counters, int total) {
return calculateSummaryStatusOfStage(counters, total, false);
}
@@ -428,8 +456,8 @@ public class CalculatedStatus {
*
* @return summary request status based on statuses of tasks in different states.
*/
- private static HostRoleStatus calculateSummaryDisplayStatus(Map<HostRoleStatus, Integer> counters,
- int total, boolean skippable) {
+ protected static HostRoleStatus calculateSummaryDisplayStatus(
+ Map<HostRoleStatus, Integer> counters, int total, boolean skippable) {
return counters.get(HostRoleStatus.SKIPPED_FAILED) > 0 ? HostRoleStatus.SKIPPED_FAILED :
counters.get(HostRoleStatus.FAILED) > 0 ? HostRoleStatus.FAILED:
calculateSummaryStatusOfStage(counters, total, skippable);
http://git-wip-us.apache.org/repos/asf/ambari/blob/12a3cd87/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
index 8c1bc57..0690ee7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RequestResourceProvider.java
@@ -69,10 +69,10 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.topology.LogicalRequest;
import org.apache.ambari.server.topology.TopologyManager;
+import org.apache.commons.lang.StringUtils;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
-import org.apache.commons.lang.StringUtils;
/**
* Resource provider for request resources.
@@ -566,13 +566,18 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
ServiceComponentTuple that = (ServiceComponentTuple) o;
- if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null)
+ if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) {
return false;
+ }
return !(componentName != null ? !componentName.equals(that.componentName) : that.componentName != null);
}
@@ -739,15 +744,20 @@ public class RequestResourceProvider extends AbstractControllerResourceProvider
// get summaries from TopologyManager for logical requests
summary.putAll(topologyManager.getStageSummaries(entity.getRequestId()));
+ // summary might be empty due to delete host have cleared all
+ // HostRoleCommands or due to hosts haven't registered yet with the cluster
+ // when the cluster is provisioned with a Blueprint
+ final CalculatedStatus status;
LogicalRequest logicalRequest = topologyManager.getRequest(entity.getRequestId());
-
- CalculatedStatus status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
- if (summary.isEmpty() && logicalRequest == null) {
-
- // summary might be empty due to delete host have cleared all HostRoleCommands
- // or due to hosts haven't registered yet with the cluster when the cluster is provisioned
- // with a Blueprint
- status = CalculatedStatus.getCompletedStatus();
+ if (summary.isEmpty() && null != logicalRequest) {
+ // in this case, it appears that there are no tasks but this is a logical
+ // topology request, so it's a matter of hosts simply not registering yet
+ // for tasks to be created
+ status = CalculatedStatus.PENDING;
+ } else {
+ // there are either tasks or this is not a logical request, so do normal
+ // status calculations
+ status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
}
setResourceProperty(resource, REQUEST_STATUS_PROPERTY_ID, status.getStatus().toString(), requestedPropertyIds);
http://git-wip-us.apache.org/repos/asf/ambari/blob/12a3cd87/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
index 59dd9d9..a778882 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StageResourceProvider.java
@@ -339,7 +339,7 @@ public class StageResourceProvider extends AbstractControllerResourceProvider im
CalculatedStatus status;
if (summary.isEmpty()) {
// Delete host might have cleared all HostRoleCommands
- status = CalculatedStatus.getCompletedStatus();
+ status = CalculatedStatus.COMPLETED;
} else {
status = CalculatedStatus.statusFromStageSummary(summary, Collections.singleton(entity.getStageId()));
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/12a3cd87/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index d6a4bdd..d527b2d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -596,6 +596,14 @@ public class TopologyManager {
return clusterTopologyMap.get(clusterId);
}
+ /**
+ * Gets a map of components keyed by host which have operations in the
+ * {@link HostRoleStatus#PENDING} state. This could either be because hosts
+ * have not registered or becuase the operations are actually waiting to be
+ * queued.
+ *
+ * @return a mapping of host with pending components.
+ */
public Map<String, Collection<String>> getPendingHostComponents() {
ensureInitialized();
Map<String, Collection<String>> hostComponentMap = new HashMap<String, Collection<String>>();
@@ -603,7 +611,16 @@ public class TopologyManager {
for (LogicalRequest logicalRequest : allRequests.values()) {
Map<Long, HostRoleCommandStatusSummaryDTO> summary = logicalRequest.getStageSummaries();
final CalculatedStatus status = CalculatedStatus.statusFromStageSummary(summary, summary.keySet());
- if (status.getStatus().isInProgress()) {
+
+ // either use the calculated status of the stage or the fact that there
+ // are no tasks and the request has no end time to determine if the
+ // request is still in progress
+ boolean logicalRequestInProgress = false;
+ if (status.getStatus().isInProgress() || (summary.isEmpty() && logicalRequest.getEndTime() <= 0) ) {
+ logicalRequestInProgress = true;
+ }
+
+ if (logicalRequestInProgress) {
Map<String, Collection<String>> requestTopology = logicalRequest.getProjectedTopology();
for (Map.Entry<String, Collection<String>> entry : requestTopology.entrySet()) {
String host = entry.getKey();
http://git-wip-us.apache.org/repos/asf/ambari/blob/12a3cd87/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
index 6f592cd..a96f395 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CalculatedStatusTest.java
@@ -657,6 +657,37 @@ public class CalculatedStatusTest {
assertEquals(HostRoleStatus.IN_PROGRESS, calc.getStatus());
}
+ /**
+ * Tests that when there are no tasks and all counts are 0, that the returned
+ * status is {@link HostRoleStatus#COMPLETED}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testGetCompletedStatusForNoTasks() throws Exception {
+ // no status / no tasks
+ CalculatedStatus status = CalculatedStatus.statusFromTaskEntities(
+ new ArrayList<HostRoleCommandEntity>(), false);
+
+ assertEquals(HostRoleStatus.COMPLETED, status.getStatus());
+
+ // empty summaries
+ status = CalculatedStatus.statusFromStageSummary(
+ new HashMap<Long, HostRoleCommandStatusSummaryDTO>(), new HashSet<Long>());
+
+ assertEquals(HostRoleStatus.COMPLETED, status.getStatus());
+
+ // generate a map of 0's - COMPLETED=0, IN_PROGRESS=0, etc
+ Map<HostRoleStatus, Integer> counts = CalculatedStatus.calculateStatusCounts(new ArrayList<HostRoleStatus>());
+ Map<HostRoleStatus, Integer> displayCounts = CalculatedStatus.calculateStatusCounts(new ArrayList<HostRoleStatus>());
+
+ HostRoleStatus hostRoleStatus = CalculatedStatus.calculateSummaryStatusOfUpgrade(counts, 0);
+ HostRoleStatus hostRoleDisplayStatus = CalculatedStatus.calculateSummaryDisplayStatus(displayCounts, 0, false);
+
+ assertEquals(HostRoleStatus.COMPLETED, hostRoleStatus);
+ assertEquals(HostRoleStatus.COMPLETED, hostRoleDisplayStatus);
+ }
+
private Collection<HostRoleCommandEntity> getTaskEntities(HostRoleStatus... statuses) {
Collection<HostRoleCommandEntity> entities = new LinkedList<HostRoleCommandEntity>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/12a3cd87/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
index 5dfc74d..f7dff11 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java
@@ -20,11 +20,10 @@ package org.apache.ambari.server.controller.internal;
import static org.apache.ambari.server.controller.internal.HostComponentResourceProvider.HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID;
-import org.apache.ambari.server.topology.Blueprint;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.newCapture;
import static org.powermock.api.easymock.PowerMock.createMock;
import static org.powermock.api.easymock.PowerMock.createNiceMock;
@@ -77,6 +76,7 @@ import org.apache.ambari.server.security.authorization.AuthorizationHelperInitia
import org.apache.ambari.server.security.authorization.RoleAuthorization;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.HostGroupInfo;
import org.apache.ambari.server.topology.LogicalRequest;
@@ -1200,7 +1200,7 @@ public class RequestResourceProviderTest {
}
}
Assert.assertNotNull(propertyIdToAssert);
- Assert.assertEquals("true", (String) propertyValueToAssert);
+ Assert.assertEquals("true", propertyValueToAssert);
}
@Test
@@ -1624,6 +1624,12 @@ public class RequestResourceProviderTest {
verify(managementController, actionManager, clusters, requestMock, requestDAO, hrcDAO);
}
+ /**
+ * Tests that topology requests return different status (PENDING) if there are
+ * no tasks. Normal requests should return COMPLETED.
+ *
+ * @throws Exception
+ */
@Test
@PrepareForTest(AmbariServer.class)
public void testGetLogicalRequestStatusWithNoTasks() throws Exception {
[37/50] [abbrv] ambari git commit: AMBARI-19092. HDP 3.0 support for
ZK with metainfo (alejandro)
Posted by nc...@apache.org.
AMBARI-19092. HDP 3.0 support for ZK with metainfo (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aed98397
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aed98397
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aed98397
Branch: refs/heads/branch-dev-patch-upgrade
Commit: aed983973034114931aca387f92bae5ca0d4e169
Parents: 757267f
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Fri Dec 2 13:26:49 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Tue Dec 6 16:52:32 2016 -0800
----------------------------------------------------------------------
.../HDP/3.0/services/ZOOKEEPER/metainfo.xml | 54 ++++++++++++++++++++
1 file changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/aed98397/ambari-server/src/main/resources/stacks/HDP/3.0/services/ZOOKEEPER/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/ZOOKEEPER/metainfo.xml
new file mode 100644
index 0000000..5d4fb4d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/ZOOKEEPER/metainfo.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ZOOKEEPER</name>
+ <displayName>ZooKeeper</displayName>
+ <comment>Centralized service which provides highly reliable distributed coordination</comment>
+ <version>3.4.6.3.0</version>
+ <extends>common-services/ZOOKEEPER/3.4.6</extends>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>amazon2015,redhat6,redhat7,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>zookeeper_${stack_version}</name>
+ </package>
+ <package>
+ <name>zookeeper_${stack_version}-server</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>zookeeper-${stack_version}</name>
+ </package>
+ <package>
+ <name>zookeeper-${stack_version}-server</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ </service>
+ </services>
+</metainfo>
[11/50] [abbrv] ambari git commit: AMBARI-19058. Perf: Deploy 3000
Agent cluster and find perf bugs. Part 1.(vbrodetskyi)
Posted by nc...@apache.org.
AMBARI-19058. Perf: Deploy 3000 Agent cluster and find perf bugs. Part 1.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/08342b3c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/08342b3c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/08342b3c
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 08342b3cb268481e5033c99cbf418f123e509327
Parents: cac20e0
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Mon Dec 5 15:12:32 2016 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Mon Dec 5 15:12:32 2016 +0200
----------------------------------------------------------------------
contrib/utils/perf/deploy-gce-perf-cluster.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/08342b3c/contrib/utils/perf/deploy-gce-perf-cluster.py
----------------------------------------------------------------------
diff --git a/contrib/utils/perf/deploy-gce-perf-cluster.py b/contrib/utils/perf/deploy-gce-perf-cluster.py
index 4737c6f..6de3938 100644
--- a/contrib/utils/perf/deploy-gce-perf-cluster.py
+++ b/contrib/utils/perf/deploy-gce-perf-cluster.py
@@ -28,7 +28,7 @@ import re
import socket
cluster_prefix = "perf"
-ambari_repo_file_url = "http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos6/2.x/latest/trunk/ambaribn.repo"
+ambari_repo_file_url = "http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos6/2.x/updates/2.5.0.0/ambaribn.repo"
public_hostname_script = "foo"
hostname_script = "foo"
@@ -285,8 +285,8 @@ def create_vms(args, number_of_nodes):
time.sleep(10)
# trying to create cluster with needed params
- print "Creating agent VMs {0}-agent-{1} with {2} large nodes on centos6...".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes))
- execute_command(args, args.controller, "/usr/sbin/gce up {0}-agent-{1} {2} --centos6 --large".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes)),
+ print "Creating agent VMs {0}-agent-{1} with {2} xlarge nodes on centos6...".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes))
+ execute_command(args, args.controller, "/usr/sbin/gce up {0}-agent-{1} {2} --centos6 --xlarge".format(cluster_prefix, args.cluster_suffix, str(number_of_nodes)),
"Failed to create cluster VMs, probably not enough resources!", "-tt")
# VMs are not accessible immediately
@@ -331,7 +331,7 @@ def create_server_script(server_host_name):
contents = "#!/bin/bash\n" + \
"wget -O /etc/yum.repos.d/ambari.repo {0}\n".format(ambari_repo_file_url) + \
"yum clean all; yum install git ambari-server -y\n" + \
- "mkdir /home ; cd /home ; git clone https://github.com/apache/ambari.git\n" + \
+ "mkdir /home ; cd /home ; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5\n" + \
"cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-server/resources/stacks/PERF\n" + \
"cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF\n" + \
"\n" + \
@@ -384,7 +384,7 @@ def create_agent_script(server_host_name):
contents = "#!/bin/bash\n" + \
"wget -O /etc/yum.repos.d/ambari.repo {0}\n".format(ambari_repo_file_url) + \
"yum clean all; yum install git ambari-agent -y\n" + \
- "mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git\n" + \
+ "mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5\n" + \
"cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF\n" + \
"sed -i -e 's/hostname=localhost/hostname={0}/g' /etc/ambari-agent/conf/ambari-agent.ini\n".format(server_host_name) + \
"sed -i -e 's/agent]/agent]\\nhostname_script={0}\\npublic_hostname_script={1}\\n/1' /etc/ambari-agent/conf/ambari-agent.ini\n".format(hostname_script, public_hostname_script) + \
[13/50] [abbrv] ambari git commit: AMBARI-19059 Falcon quicklinks
combobox is absent (dsen)
Posted by nc...@apache.org.
AMBARI-19059 Falcon quicklinks combobox is absent (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/431d2659
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/431d2659
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/431d2659
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 431d2659c67f7c07fe704b67eded4f83fe30d25a
Parents: 89b9cf5
Author: Dmytro Sen <ds...@apache.org>
Authored: Mon Dec 5 17:25:35 2016 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Mon Dec 5 17:25:35 2016 +0200
----------------------------------------------------------------------
.../FALCON/0.5.0.2.1/metainfo.xml | 7 ++++
.../FALCON/0.5.0.2.1/quicklinks/quicklinks.json | 35 ++++++++++++++++++++
2 files changed, 42 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/431d2659/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml
index 7dab615..2cda70a 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml
@@ -131,6 +131,13 @@
<excluded-config-types>
<config-type>oozie-site</config-type>
</excluded-config-types>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/431d2659/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/quicklinks/quicklinks.json
new file mode 100644
index 0000000..cc81fca
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/quicklinks/quicklinks.json
@@ -0,0 +1,35 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"https",
+ "checks":[
+ {
+ "property":"falcon.enableTLS",
+ "desired":"true",
+ "site":"falcon-startup-properties"
+ }
+ ]
+ },
+
+ "links": [
+ {
+ "name": "falcon_dashboard",
+ "label": "Falcon Dashboard",
+ "requires_user_name": "true",
+ "component_name": "FALCON_SERVER",
+ "url": "%@://%@:%@/",
+ "port":{
+ "http_property": "falcon_port",
+ "http_default_port": "15000",
+ "https_property": "falcon_port",
+ "https_default_port": "15443",
+ "regex": "^(\\d+)$",
+ "site": "falcon-env"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
[19/50] [abbrv] ambari git commit: AMBARI-19074. Host-Ordered: Empty
groups are returned for Downgrade (ncole)
Posted by nc...@apache.org.
AMBARI-19074. Host-Ordered: Empty groups are returned for Downgrade (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/50ead4d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/50ead4d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/50ead4d2
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 50ead4d237b0c03fee68192a5983044409e3e9db
Parents: 8fbc271
Author: Nate Cole <nc...@hortonworks.com>
Authored: Mon Dec 5 12:54:41 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Mon Dec 5 15:27:19 2016 -0500
----------------------------------------------------------------------
.../ambari/server/state/stack/UpgradePack.java | 19 ++++++++--
.../ambari/server/state/UpgradeHelperTest.java | 39 +++++++++++---------
2 files changed, 36 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/50ead4d2/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
index aa97fa3..76f1897 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
@@ -289,10 +289,15 @@ public class UpgradePack {
if (direction.isUpgrade()) {
list = groups;
} else {
- if (type == UpgradeType.ROLLING) {
- list = getDowngradeGroupsForRolling();
- } else if (type == UpgradeType.NON_ROLLING) {
- list = getDowngradeGroupsForNonrolling();
+ switch (type) {
+ case NON_ROLLING:
+ list = getDowngradeGroupsForNonrolling();
+ break;
+ case HOST_ORDERED:
+ case ROLLING:
+ default:
+ list = getDowngradeGroupsForRolling();
+ break;
}
}
@@ -358,6 +363,12 @@ public class UpgradePack {
private List<Grouping> getDowngradeGroupsForRolling() {
List<Grouping> reverse = new ArrayList<Grouping>();
+ // !!! Testing exposed groups.size() == 1 issue. Normally there's no precedent for
+ // a one-group upgrade pack, so take it into account anyway.
+ if (groups.size() == 1) {
+ return groups;
+ }
+
int idx = 0;
int iter = 0;
Iterator<Grouping> it = groups.iterator();
http://git-wip-us.apache.org/repos/asf/ambari/blob/50ead4d2/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 6b4b85a..ea1f18a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -1921,25 +1922,27 @@ public class UpgradeHelperTest {
ServiceComponentHost sch1 = sc.addServiceComponentHost("h1");
ServiceComponentHost sch2 = sc.addServiceComponentHost("h2");
- UpgradePack upgradePack = new UpgradePack() {
- @Override
- public List<Grouping> getGroups(Direction direction) {
- HostOrderItem hostItem = new HostOrderItem(HostOrderActionType.HOST_UPGRADE,
- Lists.newArrayList("h1", "h2"));
- HostOrderItem checkItem = new HostOrderItem(HostOrderActionType.SERVICE_CHECK,
- Lists.newArrayList("ZOOKEEPER", "STORM"));
-
- Grouping g = new HostOrderGrouping();
- ((HostOrderGrouping) g).setHostOrderItems(Lists.newArrayList(hostItem, checkItem));
- g.title = "Some Title";
- return Lists.newArrayList(g);
- }
+ // !!! make a custom grouping
+ HostOrderItem hostItem = new HostOrderItem(HostOrderActionType.HOST_UPGRADE,
+ Lists.newArrayList("h1", "h2"));
+ HostOrderItem checkItem = new HostOrderItem(HostOrderActionType.SERVICE_CHECK,
+ Lists.newArrayList("ZOOKEEPER", "STORM"));
+
+ Grouping g = new HostOrderGrouping();
+ ((HostOrderGrouping) g).setHostOrderItems(Lists.newArrayList(hostItem, checkItem));
+ g.title = "Some Title";
+
+ UpgradePack upgradePack = new UpgradePack();
+
+ // !!! set the groups directly; allow the logic in getGroups(Direction) to happen
+ Field field = UpgradePack.class.getDeclaredField("groups");
+ field.setAccessible(true);
+ field.set(upgradePack, Lists.newArrayList(g));
+
+ field = UpgradePack.class.getDeclaredField("type" );
+ field.setAccessible(true);
+ field.set(upgradePack, UpgradeType.HOST_ORDERED);
- @Override
- public Map<String, Map<String, ProcessingComponent>> getTasks() {
- return new HashMap<>();
- }
- };
MasterHostResolver resolver = new MasterHostResolver(m_configHelper, c);
UpgradeContext context = new UpgradeContext(c, UpgradeType.HOST_ORDERED, Direction.UPGRADE, new HashMap<String, Object>());
[33/50] [abbrv] ambari git commit: AMBARI-19107. False warning about
NTP not running on SUSE12. (Attila Doroszlai via swagle)
Posted by nc...@apache.org.
AMBARI-19107. False warning about NTP not running on SUSE12. (Attila Doroszlai via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/64188616
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/64188616
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/64188616
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 641886160877e4bc4c38e9d461967285d77bd0c3
Parents: 502d444
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Tue Dec 6 12:50:18 2016 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Tue Dec 6 12:50:18 2016 -0800
----------------------------------------------------------------------
ambari-agent/src/main/python/ambari_agent/HostInfo.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/64188616/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index 338daac..3810b52 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -121,12 +121,12 @@ class HostInfo(object):
return False
def get_ntp_service():
- if OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7:
- return ("chronyd", "ntpd",)
- elif OSCheck.is_redhat_family():
- return ("ntpd",)
- elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
- return ("ntp",)
+ if OSCheck.is_redhat_family():
+ return ("ntpd", "chronyd",)
+ elif OSCheck.is_suse_family():
+ return ("ntpd", "ntp",)
+ elif OSCheck.is_ubuntu_family():
+ return ("ntp", "chrony",)
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
[03/50] [abbrv] ambari git commit: AMBARI-19071. Ambari Server Unit
Test failure on branch-2.5 and trunk:
org.apache.ambari.server.state.ServicePropertiesTest.validatePropertySchemaOfServiceXMLs.(vbrodetskyi)
Posted by nc...@apache.org.
AMBARI-19071. Ambari Server Unit Test failure on branch-2.5 and trunk: org.apache.ambari.server.state.ServicePropertiesTest.validatePropertySchemaOfServiceXMLs.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c411baa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c411baa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c411baa
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 6c411baa2f3d9bae5ba7e781ac3b4cddbf18ae82
Parents: db46822
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Sat Dec 3 01:53:32 2016 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Sat Dec 3 01:53:32 2016 +0200
----------------------------------------------------------------------
.../1.0/services/HAPPY/configuration/happy-alert-config.xml | 5 +++++
.../1.0/services/HBASE/configuration/hbase-alert-config.xml | 5 +++++
.../PERF/1.0/services/HDFS/configuration/hdfs-alert-config.xml | 5 +++++
.../1.0/services/SLEEPY/configuration/sleepy-alert-config.xml | 5 +++++
.../PERF/1.0/services/SNOW/configuration/snow-alert-config.xml | 5 +++++
.../PERF/1.0/services/YARN/configuration/yarn-alert-config.xml | 5 +++++
.../1.0/services/ZOOKEEPER/configuration/zk-alert-config.xml | 5 +++++
7 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/HAPPY/configuration/happy-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/HAPPY/configuration/happy-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/HAPPY/configuration/happy-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/HAPPY/configuration/happy-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/HAPPY/configuration/happy-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/HBASE/configuration/hbase-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/HBASE/configuration/hbase-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/HBASE/configuration/hbase-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/HBASE/configuration/hbase-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/HBASE/configuration/hbase-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/HDFS/configuration/hdfs-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/HDFS/configuration/hdfs-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/HDFS/configuration/hdfs-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/HDFS/configuration/hdfs-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/HDFS/configuration/hdfs-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/SLEEPY/configuration/sleepy-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/SLEEPY/configuration/sleepy-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/SLEEPY/configuration/sleepy-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/SLEEPY/configuration/sleepy-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/SLEEPY/configuration/sleepy-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/SNOW/configuration/snow-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/SNOW/configuration/snow-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/SNOW/configuration/snow-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/SNOW/configuration/snow-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/SNOW/configuration/snow-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/configuration/yarn-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/configuration/yarn-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/configuration/yarn-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/configuration/yarn-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/YARN/configuration/yarn-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c411baa/ambari-server/src/main/resources/stacks/PERF/1.0/services/ZOOKEEPER/configuration/zk-alert-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/ZOOKEEPER/configuration/zk-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/ZOOKEEPER/configuration/zk-alert-config.xml
index 760e706..392eea7 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/ZOOKEEPER/configuration/zk-alert-config.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/ZOOKEEPER/configuration/zk-alert-config.xml
@@ -26,6 +26,7 @@
This property describes type of alert behaviour.
There are three types percentage, timeout, flip.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -37,6 +38,7 @@
set to "percentage". Here you should set percent of successful
alert checks.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -48,6 +50,7 @@
set to "timeout". Here you should set result which alert will
return after timeout, false|true|none.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
@@ -58,6 +61,7 @@
set to "timeout". Here you should set number of seconds for
alert to sleep.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
@@ -69,6 +73,7 @@
set to "flip". Here you should set number of minutes at which
the alert should flip from true|false.
</description>
+ <on-ambari-upgrade add="false"/>
</property>
[24/50] [abbrv] ambari git commit: AMBARI-18965 : Investigate and
reduce AMS HA dependency on Zookeeper. (avijayan)
Posted by nc...@apache.org.
AMBARI-18965 : Investigate and reduce AMS HA dependency on Zookeeper. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e88f8464
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e88f8464
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e88f8464
Branch: refs/heads/branch-dev-patch-upgrade
Commit: e88f84646d587549d00ef83496f4e9b61572d2cf
Parents: c711850
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Dec 5 14:30:53 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Mon Dec 5 14:30:53 2016 -0800
----------------------------------------------------------------------
.../timeline/AbstractTimelineMetricsSink.java | 40 ++++++++++++++++----
.../availability/MetricCollectorHAHelper.java | 1 -
.../availability/MetricCollectorHATest.java | 1 -
.../src/main/python/core/config_reader.py | 2 +-
.../timeline/HBaseTimelineMetricStore.java | 18 ++++-----
.../timeline/TimelineMetricConfiguration.java | 14 +++++++
.../aggregators/AbstractTimelineAggregator.java | 6 +--
.../MetricCollectorHAController.java | 4 +-
.../TestApplicationHistoryServer.java | 14 ++++++-
.../MetricCollectorHAControllerTest.java | 4 +-
.../0.1.0/configuration/ams-env.xml | 2 +-
.../0.1.0/configuration/ams-site.xml | 12 ++++++
.../0.1.0/package/scripts/params.py | 13 ++++---
13 files changed, 96 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
index 2c0c822..f92f968 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java
@@ -84,6 +84,10 @@ public abstract class AbstractTimelineMetricsSink {
public int ZK_CONNECT_TRY_COUNT = 10;
public int ZK_SLEEP_BETWEEN_RETRY_TIME = 2000;
public boolean shardExpired = true;
+ private int zookeeperMinBackoffTimeMins = 2;
+ private int zookeeperMaxBackoffTimeMins = 5;
+ private long zookeeperBackoffTimeMillis;
+ private long lastFailedZkRequestTime = 0l;
private SSLSocketFactory sslSocketFactory;
@@ -132,6 +136,7 @@ public abstract class AbstractTimelineMetricsSink {
metricSinkWriteShardStrategy = new MetricSinkWriteShardHostnameHashingStrategy(getHostname());
collectorHAHelper = new MetricCollectorHAHelper(getZookeeperQuorum(),
ZK_CONNECT_TRY_COUNT, ZK_SLEEP_BETWEEN_RETRY_TIME);
+ zookeeperBackoffTimeMillis = getZookeeperBackoffTimeMillis();
isInitializedForHA = true;
}
@@ -337,13 +342,27 @@ public abstract class AbstractTimelineMetricsSink {
}
// Reach out to all configured collectors before Zookeeper
- refreshCollectorsFromConfigured();
+ Collection<String> collectorHosts = getConfiguredCollectorHosts();
+ refreshCollectorsFromConfigured(collectorHosts);
// Lookup Zookeeper for live hosts - max 10 seconds wait time
- if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null) {
- //TODO : Bring back Zk fallback after proper curation.
- LOG.info("No live collectors from configuration. Not requesting zookeeper...");
- //allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode());
+ long currentTime = System.currentTimeMillis();
+ if (allKnownLiveCollectors.size() == 0 && getZookeeperQuorum() != null
+ && (currentTime - lastFailedZkRequestTime) > zookeeperBackoffTimeMillis) {
+
+ LOG.info("No live collectors from configuration. Requesting zookeeper...");
+ allKnownLiveCollectors.addAll(collectorHAHelper.findLiveCollectorHostsFromZNode());
+ boolean noNewCollectorFromZk = true;
+ for (String collectorHostFromZk : allKnownLiveCollectors) {
+ if (!collectorHosts.contains(collectorHostFromZk)) {
+ noNewCollectorFromZk = false;
+ break;
+ }
+ }
+ if (noNewCollectorFromZk) {
+ LOG.info("No new collector was found from Zookeeper. Will not request zookeeper for " + zookeeperBackoffTimeMillis + " millis");
+ lastFailedZkRequestTime = System.currentTimeMillis();
+ }
}
if (allKnownLiveCollectors.size() != 0) {
@@ -356,7 +375,7 @@ public abstract class AbstractTimelineMetricsSink {
// OR
// through Expiry (Refresh needed to pick up dead collectors that might have not become alive).
if (shardExpired) {
- refreshCollectorsFromConfigured();
+ refreshCollectorsFromConfigured(getConfiguredCollectorHosts());
}
return metricSinkWriteShardStrategy.findCollectorShard(new ArrayList<>(allKnownLiveCollectors));
}
@@ -376,8 +395,7 @@ public abstract class AbstractTimelineMetricsSink {
return null;
}
- private void refreshCollectorsFromConfigured() {
- Collection<String> collectorHosts = getConfiguredCollectorHosts();
+ private void refreshCollectorsFromConfigured(Collection<String> collectorHosts) {
LOG.debug("Trying to find live collector host from : " + collectorHosts);
if (collectorHosts != null && !collectorHosts.isEmpty()) {
@@ -497,6 +515,12 @@ public abstract class AbstractTimelineMetricsSink {
return hosts;
}
+
+ private long getZookeeperBackoffTimeMillis() {
+ return (zookeeperMinBackoffTimeMins +
+ rand.nextInt(zookeeperMaxBackoffTimeMins - zookeeperMinBackoffTimeMins + 1)) * 60*1000l;
+ }
+
/**
* Get a pre-formatted URI for the collector
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
index 54c2bdf..c6f6beb 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHAHelper.java
@@ -23,7 +23,6 @@ import org.apache.curator.CuratorZookeeperClient;
import org.apache.curator.RetryLoop;
import org.apache.curator.RetryPolicy;
import org.apache.curator.retry.BoundedExponentialBackoffRetry;
-import org.apache.curator.retry.RetryUntilElapsed;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
index a0bc2e8..a393a96 100644
--- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java
@@ -49,7 +49,6 @@ import static org.powermock.api.easymock.PowerMock.verifyAll;
@PrepareForTest({AbstractTimelineMetricsSink.class, URL.class, HttpURLConnection.class, MetricCollectorHAHelper.class})
public class MetricCollectorHATest {
- @Ignore
@Test
public void findCollectorUsingZKTest() throws Exception {
InputStream is = createNiceMock(InputStream.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
index b84979a..7010187 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
@@ -223,7 +223,7 @@ class Configuration:
return self.get("collector", "failover_strategy", ROUND_ROBIN_FAILOVER_STRATEGY)
def get_failover_strategy_blacklisted_interval_seconds(self):
- return self.get("collector", "failover_strategy_blacklisted_interval_seconds", 600)
+ return self.get("collector", "failover_strategy_blacklisted_interval_seconds", 300)
def get_hostname_script(self):
if self.hostname_script:
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index a6f8e2f..5dc3aee 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -106,16 +106,14 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
// Initialize policies before TTL update
hBaseAccessor.initPoliciesAndTTL();
// Start HA service
- if (configuration.isDistributedOperationModeEnabled()) {
- // Start the controller
- haController = new MetricCollectorHAController(configuration);
- try {
- haController.initializeHAController();
- } catch (Exception e) {
- LOG.error(e);
- throw new MetricsSystemInitializationException("Unable to " +
- "initialize HA controller", e);
- }
+ // Start the controller
+ haController = new MetricCollectorHAController(configuration);
+ try {
+ haController.initializeHAController();
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new MetricsSystemInitializationException("Unable to " +
+ "initialize HA controller", e);
}
String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, "");
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 22710b0..13d361b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -323,6 +323,20 @@ public class TimelineMetricConfiguration {
return hbaseConf.getTrimmed("hbase.zookeeper.quorum");
}
+ public String getClusterZKClientPort() throws MalformedURLException, URISyntaxException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return metricsConf.getTrimmed("cluster.zookeeper.property.clientPort", "2181");
+ }
+
+ public String getClusterZKQuorum() throws MalformedURLException, URISyntaxException {
+ if (!isInitialized) {
+ initialize();
+ }
+ return metricsConf.getTrimmed("cluster.zookeeper.quorum");
+ }
+
public String getInstanceHostnameFromEnv() throws UnknownHostException {
String amsInstanceName = System.getProperty("AMS_INSTANCE_NAME");
if (amsInstanceName == null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
index 1096bc4..cb131d3 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java
@@ -317,9 +317,9 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg
protected void downsample(Connection conn, Long startTime, Long endTime) {
- LOG.info("Checking for downsampling requests.");
+ LOG.debug("Checking for downsampling requests.");
if (CollectionUtils.isEmpty(configuredDownSamplers)) {
- LOG.info("No downsamplers configured");
+ LOG.debug("No downsamplers configured");
return;
}
@@ -406,7 +406,7 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg
PreparedStatement stmt = null;
ResultSet rs = null;
- LOG.info("Downsampling query : " + condition.getStatement());
+ LOG.debug("Downsampling query : " + condition.getStatement());
try {
stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition);
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
index edce367..26702b6 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAController.java
@@ -81,8 +81,8 @@ public class MetricCollectorHAController {
}
try {
- String zkClientPort = configuration.getZKClientPort();
- String zkQuorum = configuration.getZKQuorum();
+ String zkClientPort = configuration.getClusterZKClientPort();
+ String zkQuorum = configuration.getClusterZKQuorum();
if (StringUtils.isEmpty(zkClientPort) || StringUtils.isEmpty(zkQuorum)) {
throw new Exception("Unable to parse zookeeper quorum. clientPort = "
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
index 462213d..3688630 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.HBaseTimelineMetricStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.MetricCollectorHAController;
import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultPhoenixDataSource;
import org.apache.zookeeper.ClientCnxn;
import org.easymock.EasyMock;
@@ -71,7 +73,7 @@ import static org.powermock.api.support.membermodification.MemberMatcher.method;
import static org.powermock.api.support.membermodification.MemberModifier.suppress;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ PhoenixHBaseAccessor.class, UserGroupInformation.class,
+@PrepareForTest({ PhoenixHBaseAccessor.class, HBaseTimelineMetricStore.class, UserGroupInformation.class,
ClientCnxn.class, DefaultPhoenixDataSource.class, ConnectionFactory.class,
TimelineMetricConfiguration.class, ApplicationHistoryServer.class })
@PowerMockIgnore( {"javax.management.*"})
@@ -179,6 +181,8 @@ public class TestApplicationHistoryServer {
expect(metricConfiguration.getTimelineMetricsServiceHandlerThreadCount()).andReturn(20).anyTimes();
expect(metricConfiguration.getWebappAddress()).andReturn("localhost:9990").anyTimes();
expect(metricConfiguration.getTimelineServiceRpcAddress()).andReturn("localhost:10299").anyTimes();
+ expect(metricConfiguration.getClusterZKQuorum()).andReturn("localhost").anyTimes();
+ expect(metricConfiguration.getClusterZKClientPort()).andReturn("2181").anyTimes();
Connection connection = createNiceMock(Connection.class);
Statement stmt = createNiceMock(Statement.class);
@@ -197,6 +201,14 @@ public class TestApplicationHistoryServer {
connection.close();
expectLastCall();
+ MetricCollectorHAController haControllerMock = PowerMock.createMock(MetricCollectorHAController.class);
+ expectNew(MetricCollectorHAController.class, metricConfiguration)
+ .andReturn(haControllerMock);
+
+ haControllerMock.initializeHAController();
+ expectLastCall().once();
+ expect(haControllerMock.isInitialized()).andReturn(false).anyTimes();
+
org.apache.hadoop.hbase.client.Connection conn = createNiceMock(org.apache.hadoop.hbase.client.Connection.class);
mockStatic(ConnectionFactory.class);
expect(ConnectionFactory.createConnection((Configuration) anyObject())).andReturn(conn);
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
index 1e4bac0..875f013 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/availability/MetricCollectorHAControllerTest.java
@@ -51,8 +51,8 @@ public class MetricCollectorHAControllerTest extends AbstractMiniHBaseClusterTes
String port = zkUrl.split(":")[3];
String quorum = zkUrl.split(":")[2];
- expect(configuration.getZKClientPort()).andReturn(port);
- expect(configuration.getZKQuorum()).andReturn(quorum);
+ expect(configuration.getClusterZKClientPort()).andReturn(port);
+ expect(configuration.getClusterZKQuorum()).andReturn(quorum);
replay(configuration);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
index 48232af..a33656e 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
@@ -82,7 +82,7 @@
</property>
<property>
<name>failover_strategy_blacklisted_interval</name>
- <value>600</value>
+ <value>300</value>
<description>
Metrics collector host will be blacklisted for specified number of seconds if metric monitor failed to connect to it.
</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index 9235777..45fe1a7 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -783,4 +783,16 @@
</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>cluster.zookeeper.quorum</name>
+ <value>{{cluster_zookeeper_quorum_hosts}}</value>
+ <description>Comma separated list of servers in the cluster ZooKeeper Quorum.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>cluster.zookeeper.property.clientPort</name>
+ <value>{{cluster_zookeeper_clientPort}}</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e88f8464/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 529cf4c..5ab6a4c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -240,15 +240,18 @@ else:
max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_limit", "32768")
hostname = config["hostname"]
+cluster_zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts'])
+if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ cluster_zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+else:
+ cluster_zookeeper_clientPort = '2181'
+
if not is_hbase_distributed:
zookeeper_quorum_hosts = hostname
zookeeper_clientPort = '61181'
else:
- zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts'])
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
+ zookeeper_quorum_hosts = cluster_zookeeper_quorum_hosts
+ zookeeper_clientPort = cluster_zookeeper_clientPort
ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir']
_hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir']
[10/50] [abbrv] ambari git commit: AMBARI-19067. Special characters
break creating user homes (Laszlo Puskas via magyari_sandor)
Posted by nc...@apache.org.
AMBARI-19067. Special characters break creating user homes (Laszlo Puskas via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cac20e09
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cac20e09
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cac20e09
Branch: refs/heads/branch-dev-patch-upgrade
Commit: cac20e090151785edf2b8176d9ac0bddb1adc86f
Parents: 7f6ca9d
Author: Laszlo Puskas <lp...@hortonworks.com>
Authored: Mon Dec 5 14:07:33 2016 +0100
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Mon Dec 5 14:07:33 2016 +0100
----------------------------------------------------------------------
.../src/main/resources/scripts/post-user-creation-hook.sh | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cac20e09/ambari-server/src/main/resources/scripts/post-user-creation-hook.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/post-user-creation-hook.sh b/ambari-server/src/main/resources/scripts/post-user-creation-hook.sh
index 34169c1..ee8d2d1 100755
--- a/ambari-server/src/main/resources/scripts/post-user-creation-hook.sh
+++ b/ambari-server/src/main/resources/scripts/post-user-creation-hook.sh
@@ -84,6 +84,10 @@ echo "[" | cat > "$JSON_INPUT"
while read -r LINE
do
USR_NAME=$(echo "$LINE" | awk -F, '{print $1}')
+ echo "Processing user name: $USR_NAME"
+
+ # encoding the username
+ USR_NAME=$(printf "%q" "$USR_NAME")
cat <<EOF >> "$JSON_INPUT"
{
@@ -97,7 +101,10 @@ do
EOF
done <"$CSV_FILE"
+# deleting the last line
sed -i '$ d' "$JSON_INPUT"
+
+# appending json closing elements to the end of the file
echo $'}\n]' | cat >> "$JSON_INPUT"
echo "Generating file $JSON_INPUT ... DONE."
echo "Processing post user creation hook payload ... DONE."
[40/50] [abbrv] ambari git commit: AMBARI-19094. HDP 3.0 support for
YARN/MR with configs, kerberos, widgets, metrics, quicklinks,
and themes (alejandro)
Posted by nc...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
new file mode 100644
index 0000000..e33b91d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
@@ -0,0 +1,814 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <!-- These configs were inherited from HDP 2.1 -->
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
+ <description>Indicate to clients whether timeline service is enabled or not.
+ If enabled, clients will put entities and events to the timeline server.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.generic-application-history.store-class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <description>
+ Store class name for history store, defaulting to file system store
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.webapp.address</name>
+ <value>localhost:8188</value>
+ <description>
+ The http address of the timeline service web application.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.webapp.https.address</name>
+ <value>localhost:8190</value>
+ <description>
+ The http address of the timeline service web application.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.address</name>
+ <value>localhost:10200</value>
+ <description>
+ This is default address for the timeline server to start
+ the RPC server.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <description>Time to live for timeline store data in milliseconds.</description>
+ <name>yarn.timeline-service.ttl-ms</name>
+ <value>2678400000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <description>Length of time to wait between deletion cycles of leveldb timeline store in milliseconds.</description>
+ <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
+ <value>300000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
+ <description>Classpath for typical applications.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ <description>
+ Is the registry enabled: does the RM start it up, create the user and system paths, and purge service records when containers, application attempts and applications complete
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ <description>
+ List of hostname:port pairs defining the zookeeper quorum binding for the registry
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.recovery.enabled</name>
+ <value>true</value>
+ <description>Enable the node manager to recover after starting</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.recovery.dir</name>
+ <value>{{yarn_log_dir_prefix}}/nodemanager/recovery-state</value>
+ <description>
+ The local filesystem directory in which the node manager will store
+ state when recovery is enabled.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
+ <value>10000</value>
+ <description>Time interval between each attempt to connect to NM</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.client.nodemanager-connect.max-wait-ms</name>
+ <value>60000</value>
+ <description>Max time to wait to establish a connection to NM</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.recovery.enabled</name>
+ <value>true</value>
+ <description>
+ Enable RM to recover state after starting.
+ If true, then yarn.resourcemanager.store.class must be specified.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
+ <value>true</value>
+ <description>
+ Enable RM work preserving recovery. This configuration is private to YARN for experimenting the feature.
+ </description>
+ <display-name>Enable Work Preserving Restart</display-name>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.store.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
+ <description>
+ The class to use as the persistent store.
+ If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore is used,
+ the store is implicitly fenced; meaning a single ResourceManager
+ is able to use the store at any point in time.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-address</name>
+ <value>localhost:2181</value>
+ <description>
+ List Host:Port of the ZooKeeper servers to be used by the RM. comma separated host:port pairs, each corresponding to a zk server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" If the optional chroot suffix is used the example would look like: "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002/app/a" where the client would be rooted at "/app/a" and all paths would be relative to this root - ie getting/setting/etc... "/foo/bar" would result in operations being run on "/app/a/foo/bar" (from the server perspective).
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-state-store.parent-path</name>
+ <value>/rmstore</value>
+ <description>Full path of the ZooKeeper znode where RM state will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore as the value for yarn.resourcemanager.store.class</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-acl</name>
+ <value>world:anyone:rwcda</value>
+ <description>ACL's to be used for ZooKeeper znodes.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
+ <value>10000</value>
+ <description>Set the amount of time RM waits before allocating new containers on work-preserving-recovery. Such wait period gives RM a chance to settle down resyncing with NMs in the cluster on recovery, before assigning new containers to applications.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+ <value>30000</value>
+ <description>How often to try connecting to the ResourceManager.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.connect.max-wait.ms</name>
+ <value>900000</value>
+ <description>Maximum time to wait to establish connection to ResourceManager</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <value>1000</value>
+ <description>"Retry interval in milliseconds when connecting to ZooKeeper.
+ When HA is enabled, the value here is NOT used. It is generated
+ automatically from yarn.resourcemanager.zk-timeout-ms and
+ yarn.resourcemanager.zk-num-retries."
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-num-retries</name>
+ <value>1000</value>
+ <description>Number of times RM tries to connect to ZooKeeper.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-timeout-ms</name>
+ <value>10000</value>
+ <description>ZooKeeper session timeout in milliseconds. Session expiration is managed by the ZooKeeper cluster itself, not by the client. This value is used by the cluster to determine when the client's session expires. Expirations happens when the cluster does not hear from the client within the specified session timeout period (i.e. no heartbeat).</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.state-store.max-completed-applications</name>
+ <value>${yarn.resourcemanager.max-completed-applications}</value>
+ <description>The maximum number of completed applications RM state store keeps, less than or equals to ${yarn.resourcemanager.max-completed-applications}. By default, it equals to ${yarn.resourcemanager.max-completed-applications}. This ensures that the applications kept in the state store are consistent with the applications remembered in RM memory. Any values larger than ${yarn.resourcemanager.max-completed-applications} will be reset to ${yarn.resourcemanager.max-completed-applications}. Note that this value impacts the RM recovery performance.Typically, a smaller value indicates better performance on RM recovery.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
+ <value>2000, 500</value>
+ <description>hdfs client retry policy specification. hdfs client retry is always enabled. Specified in pairs of sleep-time and number-of-retries and (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on average, the following n1 retries sleep t1 milliseconds on average, and so on.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.uri</name>
+ <value> </value>
+ <description>RI pointing to the location of the FileSystem path where RM state will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore as the value for yarn.resourcemanager.store.class </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.ha.enabled</name>
+ <value>false</value>
+ <description>enable RM HA or not</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
+ <description>Pre-requisite to use CGroups</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+ <value>hadoop-yarn</value>
+ <description>Name of the Cgroups hierarchy under which all YARN jobs will be launched</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+ <value>false</value>
+ <description>If true, YARN will automount the CGroup, however the directory needs to already exist; else, the cgroup should be mounted by the admin</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
+ <value>/cgroup</value>
+ <description>Path used by the LCE to mount cgroups if not found. This path must exist before the NodeManager is launched.</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
+ <value>false</value>
+ <description>Strictly limit CPU resource usage to allocated usage even if spare CPU is available</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>8</value>
+ <description>Number of vcores that can be allocated
+ for containers. This is used by the RM scheduler when allocating
+ resources for containers. This is not used to limit the number of
+ CPUs used by YARN containers. If it is set to -1 and
+ yarn.nodemanager.resource.detect-hardware-capabilities is true, it is
+ automatically determined from the hardware in case of Windows and Linux.
+ In other cases, number of vcores is 8 by default.
+ </description>
+ <display-name>Number of virtual cores</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>32</maximum>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+ <value>80</value>
+ <description>The amount of CPU allocated for YARN containers - only effective when used with CGroups</description>
+ <display-name>Percentage of physical CPU allocated for all containers on a node</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>100</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.node-labels.fs-store.retry-policy-spec</name>
+ <value>2000, 500</value>
+ <description>
+ Retry policy used for FileSystem node label store. The policy is
+ specified by N pairs of sleep-time in milliseconds and number-of-retries
+ "s1,n1,s2,n2,...".
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
+ <value>1000</value>
+ <description>This is related to disk size on the machines, admins should set one of yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb or yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage but not both. If both are set, the more conservative value will be used</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>90</value>
+ <description>This is related to disk size on the machines, admins should set one of yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb or yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage but not both. If both are set, the more conservative value will be used</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+ <value>-1</value>
+ <description>Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded whenthe application is finished. By setting this configure, logs can be uploaded periodically when the application is running. The minimum rolling-interval-seconds can be set is 3600.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
+ <value>false</value>
+ <description>
+ This configuration is for debug and test purpose.
+ By setting this configuration as true.
+ We can break the lower bound of yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+ <value>30</value>
+ <description>This is temporary solution. The configuration will be deleted once, we find a more scalable method to only write a single log file per LRS.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
+ <value>10</value>
+ <description>Number of worker threads that send the yarn system metrics data.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.client.max-retries</name>
+ <value>30</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.client.retry-interval-ms</name>
+ <value>1000</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.ttl-enable</name>
+ <value>true</value>
+ <description>
+ Enable age off of timeline store data.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.state-store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
+ <description>Store class name for timeline state store.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-state-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ <description>Store file name for leveldb state store.</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ <description>Store file name for leveldb timeline store.</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
+ <value>104857600</value>
+ <description>
+ Size of read cache for uncompressed blocks for leveldb timeline store in bytes.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
+ <value>10000</value>
+ <description>
+ Size of cache for recently read entity start times for leveldb timeline store in number of entities.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <value>10000</value>
+ <description>
+ Size of cache for recently written entity start times for leveldb timeline store in number of entities.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for the Timeline Server HTTP endpoint.
+ Supported values are: simple | kerberos | $AUTHENTICATION_HANDLER_CLASSNAME
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>false</value>
+ <description>
+ Flag to enable override of the default kerberos authentication filter with
+ the RM authentication filter to allow authentication using delegation
+ tokens(fallback to kerberos if the tokens are missing).
+ Only applicable when the http authentication type is kerberos.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.bind-host</name>
+ <value>0.0.0.0</value>
+ <description>Default value is 0.0.0.0, when this is set the service will bind on all interfaces. I think these two options (blank, "0.0.0.0" sans quotes) should be the two available values, with blank as the default.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.bind-host</name>
+ <value>0.0.0.0</value>
+ <description>Default value is 0.0.0.0, when this is set the service will bind on all interfaces. I think these two options (blank, "0.0.0.0" sans quotes) should be the two available values, with blank as the default.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.bind-host</name>
+ <value>0.0.0.0</value>
+ <description>Default value is 0.0.0.0, when this is set the service will bind on all interfaces. I think these two options (blank, "0.0.0.0" sans quotes) should be the two available values, with blank as the default.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.node-labels.fs-store.root-dir</name>
+ <value>/system/yarn/node-labels</value>
+ <description>
+ URI for NodeLabelManager.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
+ <description/>
+ <display-name>Minimum Container Size (VCores)</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>8</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>8</value>
+ <description/>
+ <display-name>Maximum Container Size (VCores)</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>8</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.node-labels.enabled</name>
+ <value>false</value>
+ <description>
+ Enable node labels to restrict YARN applications so that they run only on cluster nodes that have a specified node label.
+ </description>
+ <display-name>Node Labels</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ <description>ContainerExecutor for launching containers</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ <property>
+ <type>core-site</type>
+ <name>hadoop.security.authentication</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ <description>Unix group of the NodeManager</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ <description>
+ Enable a set of periodic monitors (specified in
+ yarn.resourcemanager.scheduler.monitor.policies) that affect the
+ scheduler.
+ </description>
+ <value>false</value>
+ <display-name>Pre-emption</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- In HDP 2.3, these properties were deleted:
+ yarn.node-labels.manager-class
+ -->
+
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>yarn.timeline-service.recovery.enabled</name>
+ <description>
+ Enable timeline server to recover state after starting. If
+ true, then yarn.timeline-service.state-store-class must be specified.
+ </description>
+ <value>true</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.acl.enable</name>
+ <value>false</value>
+ <description> Are acls enabled. </description>
+ <depends-on>
+ <property>
+ <type>ranger-yarn-plugin-properties</type>
+ <name>ranger-yarn-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.authorization-provider</name>
+ <description> Yarn authorization provider class. </description>
+ <depends-on>
+ <property>
+ <type>ranger-yarn-plugin-properties</type>
+ <name>ranger-yarn-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.admin.acl</name>
+ <value>yarn</value>
+ <description> ACL of who can be admin of the YARN cluster. </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <!--ats v1.5 properties-->
+ <property>
+ <name>yarn.timeline-service.version</name>
+ <value>1.5</value>
+ <description>Timeline service version we’re currently using.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value>
+ <description>Main storage class for YARN timeline server.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
+ <value>/ats/active/</value>
+ <description>DFS path to store active application’s timeline data</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
+ <value>/ats/done/</value>
+ <description>DFS path to store done application’s timeline data</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
+ <value/>
+ <description>Plugins that can translate a timeline entity read request into a list of timeline cache ids, separated by commas. </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <!-- advanced ats v1.5 properties-->
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
+ <description>Summary storage for ATS v1.5</description>
+ <!-- Use rolling leveldb, advanced -->
+ <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
+ <description>
+ Scan interval for ATS v1.5 entity group file system storage reader.This
+ value controls how frequent the reader will scan the HDFS active directory
+ for application status.
+ </description>
+ <!-- Default is 60 seconds, advanced -->
+ <value>60</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
+ <description>
+ Scan interval for ATS v1.5 entity group file system storage cleaner.This
+ value controls how frequent the reader will scan the HDFS done directory
+ for stale application data.
+ </description>
+ <!-- 3600 is default, advanced -->
+ <value>3600</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
+ <description>
+ How long the ATS v1.5 entity group file system storage will keep an
+ application's data in the done directory.
+ </description>
+ <!-- 7 days is default, advanced -->
+ <value>604800</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.4 -->
+ <property>
+ <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
+ <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+ <description>The auxiliary service class to use for Spark</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.5 -->
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle,spark_shuffle,spark2_shuffle</value>
+ <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and cannot start with numbers</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services.spark2_shuffle.class</name>
+ <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+ <description>The auxiliary service class to use for Spark 2</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
+ <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+ <description>The auxiliary service classpath to use for Spark</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
+ <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+ <description>The auxiliary service classpath to use for Spark 2</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+ <description>Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded whenthe application is finished. By setting this configure, logs can be uploaded periodically when the application is running. The minimum rolling-interval-seconds can be set is 3600.</description>
+ <value>3600</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-metrics.unregister-delay-ms</name>
+ <value>60000</value>
+ <description>The delay time ms to unregister container metrics after completion.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath</name>
+ <value/>
+ <description>Classpath for all plugins defined in yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/kerberos.json
new file mode 100644
index 0000000..e690204
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/kerberos.json
@@ -0,0 +1,278 @@
+{
+ "services": [
+ {
+ "name": "YARN",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "configurations": [
+ {
+ "yarn-site": {
+ "yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.http-authentication.type": "kerberos",
+ "yarn.acl.enable": "true",
+ "yarn.admin.acl": "${yarn-env/yarn_user},dr.who",
+ "yarn.timeline-service.http-authentication.signature.secret": "",
+ "yarn.timeline-service.http-authentication.signature.secret.file": "",
+ "yarn.timeline-service.http-authentication.signer.secret.provider": "",
+ "yarn.timeline-service.http-authentication.signer.secret.provider.object": "",
+ "yarn.timeline-service.http-authentication.token.validity": "",
+ "yarn.timeline-service.http-authentication.cookie.domain": "",
+ "yarn.timeline-service.http-authentication.cookie.path": "",
+ "yarn.timeline-service.http-authentication.proxyusers.*.hosts": "",
+ "yarn.timeline-service.http-authentication.proxyusers.*.users": "",
+ "yarn.timeline-service.http-authentication.proxyusers.*.groups": "",
+ "yarn.timeline-service.http-authentication.kerberos.name.rules": "",
+ "yarn.resourcemanager.proxyusers.*.groups": "",
+ "yarn.resourcemanager.proxyusers.*.hosts": "",
+ "yarn.resourcemanager.proxyusers.*.users": "",
+ "yarn.resourcemanager.proxy-user-privileges.enabled": "true",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount-path": ""
+ }
+ },
+ {
+ "core-site": {
+ "hadoop.proxyuser.${yarn-env/yarn_user}.groups": "*",
+ "hadoop.proxyuser.${yarn-env/yarn_user}.hosts": "${clusterHostInfo/rm_host}"
+ }
+ },
+ {
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.root.acl_administer_queue": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.default.acl_administer_queue": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.acl_administer_jobs": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.default.acl_administer_jobs": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.default.acl_submit_applications": "${yarn-env/yarn_user}"
+ }
+ },
+ {
+ "ranger-yarn-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "NODEMANAGER",
+ "identities": [
+ {
+ "name": "nodemanager_nm",
+ "principal": {
+ "value": "nm/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.nodemanager.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/nm.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.nodemanager.keytab"
+ }
+ },
+ {
+ "name": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.service.principal"
+ },
+ "keytab": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file"
+ },
+ "when" : {
+ "contains" : ["services", "HIVE"]
+ }
+ },
+ {
+ "name": "llap_zk_hive",
+ "principal": {
+ "value": "hive/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hive-interactive-site/hive.llap.zk.sm.principal"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hive.llap.zk.sm.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file"
+ },
+ "when" : {
+ "contains" : ["services", "HIVE"]
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-keytab-file"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "yarn-site": {
+ "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"
+ }
+ }
+ ]
+ },
+ {
+ "name": "RESOURCEMANAGER",
+ "identities": [
+ {
+ "name": "resource_manager_rm",
+ "principal": {
+ "value": "rm/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.resourcemanager.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/rm.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.resourcemanager.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file"
+ }
+ },
+ {
+ "name": "/YARN/RESOURCEMANAGER/resource_manager_rm",
+ "principal": {
+ "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.principal"
+ },
+ "keytab": {
+ "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.keyTab"
+ }
+ }
+ ]
+ },
+ {
+ "name": "APP_TIMELINE_SERVER",
+ "identities": [
+ {
+ "name": "app_timeline_server_yarn",
+ "principal": {
+ "value": "yarn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.timeline-service.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.timeline-service.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "MAPREDUCE2",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "components": [
+ {
+ "name": "HISTORYSERVER",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "history_server_jhs",
+ "principal": {
+ "value": "jhs/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "mapred-site/mapreduce.jobhistory.principal",
+ "local_username": "${mapred-env/mapred_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/jhs.service.keytab",
+ "owner": {
+ "name": "${mapred-env/mapred_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "mapred-site/mapreduce.jobhistory.keytab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-principal"
+ },
+ "keytab": {
+ "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/metainfo.xml
new file mode 100644
index 0000000..7e1fd78
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/metainfo.xml
@@ -0,0 +1,173 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>YARN</name>
+ <displayName>YARN</displayName>
+ <version>2.7.1.3.0</version>
+ <extends>common-services/YARN/2.1.0.2.0</extends>
+
+ <components>
+ <component>
+ <name>APP_TIMELINE_SERVER</name>
+ <displayName>App Timeline Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>true</reassignAllowed>
+
+ <commandScript>
+ <script>scripts/application_timeline_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+
+ <dependencies>
+ <dependency>
+ <name>TEZ/TEZ_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>SPARK/SPARK_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+
+ <component>
+ <name>RESOURCEMANAGER</name>
+ <category>MASTER</category>
+ <cardinality>1-2</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+
+ <dependencies>
+ <dependency>
+ <name>TEZ/TEZ_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <configuration-dependencies>
+ <config-type>capacity-scheduler</config-type>
+ </configuration-dependencies>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>amazon2015,redhat6,redhat7,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>hadoop_${stack_version}-yarn</name>
+ </package>
+ <package>
+ <name>hadoop_${stack_version}-mapreduce</name>
+ </package>
+ <package>
+ <name>hadoop_${stack_version}-hdfs</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu12</osFamily>
+ <packages>
+ <package>
+ <name>hadoop-${stack_version}-yarn</name>
+ </package>
+ <package>
+ <name>hadoop-${stack_version}-mapreduce</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+
+ <configuration-dependencies>
+ <config-type>yarn-site</config-type>
+ <config-type>yarn-env</config-type>
+ <config-type>core-site</config-type>
+ <config-type>yarn-log4j</config-type>
+ </configuration-dependencies>
+ </service>
+
+ <service>
+ <name>MAPREDUCE2</name>
+ <displayName>MapReduce2</displayName>
+ <version>2.7.1.3.0</version>
+ <configuration-dir>configuration-mapred</configuration-dir>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>amazon2015,redhat6,redhat7,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>hadoop_${stack_version}-mapreduce</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu12</osFamily>
+ <packages>
+ <package>
+ <name>hadoop-${stack_version}-mapreduce</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <themes-dir>themes-mapred</themes-dir>
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ <quickLinksConfigurations-dir>quicklinks-mapred</quickLinksConfigurations-dir>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks-mapred/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks-mapred/quicklinks.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks-mapred/quicklinks.json
new file mode 100644
index 0000000..5ffbc07
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks-mapred/quicklinks.json
@@ -0,0 +1,80 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"https",
+ "checks":[
+ {
+ "property":"mapreduce.jobhistory.http.policy",
+ "desired":"HTTPS_ONLY",
+ "site":"mapred-site"
+ }
+ ]
+ },
+
+ "links": [
+ {
+ "name": "jobhistory_ui",
+ "label": "JobHistory UI",
+ "requires_user_name": "false",
+ "component_name": "HISTORYSERVER",
+ "url": "%@://%@:%@",
+ "port":{
+ "http_property": "mapreduce.jobhistory.webapp.address",
+ "http_default_port": "19888",
+ "https_property": "mapreduce.jobhistory.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "mapred-site"
+ }
+ },
+ {
+ "name": "jobhistory_logs",
+ "label": "JobHistory logs",
+ "requires_user_name": "false",
+ "component_name": "HISTORYSERVER",
+ "url": "%@://%@:%@/logs",
+ "port":{
+ "http_property": "mapreduce.jobhistory.webapp.address",
+ "http_default_port": "19888",
+ "https_property": "mapreduce.jobhistory.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "mapred-site"
+ }
+ },
+ {
+ "name":"jobhistory_jmx",
+ "label":"JobHistory JMX",
+ "requires_user_name":"false",
+ "component_name": "HISTORYSERVER",
+ "url":"%@://%@:%@/jmx",
+ "port":{
+ "http_property": "mapreduce.jobhistory.webapp.address",
+ "http_default_port": "19888",
+ "https_property": "mapreduce.jobhistory.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "mapred-site"
+ }
+ },
+ {
+ "name":"thread_stacks",
+ "label":"Thread Stacks",
+ "requires_user_name": "false",
+ "component_name": "HISTORYSERVER",
+ "url":"%@://%@:%@/stacks",
+ "port":{
+ "http_property": "mapreduce.jobhistory.webapp.address",
+ "http_default_port": "19888",
+ "https_property": "mapreduce.jobhistory.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "mapred-site"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks/quicklinks.json
new file mode 100644
index 0000000..37248d0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/quicklinks/quicklinks.json
@@ -0,0 +1,80 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"https",
+ "checks":[
+ {
+ "property":"yarn.http.policy",
+ "desired":"HTTPS_ONLY",
+ "site":"yarn-site"
+ }
+ ]
+ },
+
+ "links": [
+ {
+ "name": "resourcemanager_ui",
+ "label": "ResourceManager UI",
+ "requires_user_name": "false",
+ "component_name": "RESOURCEMANAGER",
+ "url": "%@://%@:%@",
+ "port":{
+ "http_property": "yarn.resourcemanager.webapp.address",
+ "http_default_port": "8088",
+ "https_property": "yarn.resourcemanager.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "yarn-site"
+ }
+ },
+ {
+ "name": "resourcemanager_logs",
+ "label": "ResourceManager logs",
+ "requires_user_name": "false",
+ "component_name": "RESOURCEMANAGER",
+ "url": "%@://%@:%@/logs",
+ "port":{
+ "http_property": "yarn.resourcemanager.webapp.address",
+ "http_default_port": "8088",
+ "https_property": "yarn.resourcemanager.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "yarn-site"
+ }
+ },
+ {
+ "name": "resourcemanager_jmx",
+ "label":"ResourceManager JMX",
+ "requires_user_name": "false",
+ "component_name": "RESOURCEMANAGER",
+ "url":"%@://%@:%@/jmx",
+ "port":{
+ "http_property": "yarn.resourcemanager.webapp.address",
+ "http_default_port": "8088",
+ "https_property": "yarn.resourcemanager.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "yarn-site"
+ }
+ },
+ {
+ "name": "thread_stacks",
+ "label":"Thread Stacks",
+ "requires_user_name": "false",
+ "component_name": "RESOURCEMANAGER",
+ "url":"%@://%@:%@/stacks",
+ "port":{
+ "http_property": "yarn.resourcemanager.webapp.address",
+ "http_default_port": "8088",
+ "https_property": "yarn.resourcemanager.webapp.https.address",
+ "https_default_port": "8090",
+ "regex": "\\w*:(\\d+)",
+ "site": "yarn-site"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes-mapred/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes-mapred/theme.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes-mapred/theme.json
new file mode 100644
index 0000000..5019447
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes-mapred/theme.json
@@ -0,0 +1,132 @@
+{
+ "name": "default",
+ "description": "Default theme for MAPREDUCE service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-mr-scheduler",
+ "display-name": "MapReduce",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "3",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-mr-scheduler-row1-col1",
+ "display-name": "MapReduce Framework",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-mr-scheduler-row1-col2",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-mr-scheduler-row1-col3",
+ "row-index": "0",
+ "column-index": "2",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-mr-scheduler-row2-col1",
+ "display-name": "MapReduce AppMaster",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "3"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "mapred-site/mapreduce.map.memory.mb",
+ "subsection-name": "subsection-mr-scheduler-row1-col1"
+ },
+ {
+ "config": "mapred-site/mapreduce.reduce.memory.mb",
+ "subsection-name": "subsection-mr-scheduler-row1-col2"
+ },
+ {
+ "config": "mapred-site/yarn.app.mapreduce.am.resource.mb",
+ "subsection-name": "subsection-mr-scheduler-row2-col1"
+ },
+ {
+ "config": "mapred-site/mapreduce.task.io.sort.mb",
+ "subsection-name": "subsection-mr-scheduler-row1-col3"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "mapred-site/mapreduce.map.memory.mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "mapred-site/mapreduce.reduce.memory.mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "mapred-site/yarn.app.mapreduce.am.resource.mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "mapred-site/mapreduce.task.io.sort.mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes/theme.json
new file mode 100644
index 0000000..758cf0c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/themes/theme.json
@@ -0,0 +1,250 @@
+{
+ "name": "default",
+ "description": "Default theme for YARN service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "3",
+ "tab-rows": "2",
+ "sections": [
+ {
+ "name": "section-nm-sizing",
+ "display-name": "Memory",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "2",
+ "section-columns": "2",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-nm-sizing-col1",
+ "display-name": "Node",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-nm-sizing-col2",
+ "display-name": "Container",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "section-yarn-platform-features",
+ "display-name": "YARN Features",
+ "row-index": "0",
+ "column-index": "2",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-yarn-platform-features-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "section-container-sizing",
+ "display-name": "CPU",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "2",
+ "section-columns": "2",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-container-sizing-col1",
+ "display-name": "Node",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-container-sizing-col2",
+ "display-name": "Container",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "yarn-site/yarn.nodemanager.resource.memory-mb",
+ "subsection-name": "subsection-nm-sizing-col1"
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.minimum-allocation-mb",
+ "subsection-name": "subsection-nm-sizing-col2"
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.maximum-allocation-mb",
+ "subsection-name": "subsection-nm-sizing-col2"
+ },
+ {
+ "config": "yarn-site/yarn.node-labels.enabled",
+ "subsection-name": "subsection-yarn-platform-features-col1"
+ },
+ {
+ "config": "yarn-site/yarn.resourcemanager.scheduler.monitor.enable",
+ "subsection-name": "subsection-yarn-platform-features-col1"
+ },
+ {
+ "config": "capacity-scheduler/yarn.scheduler.capacity.resource-calculator",
+ "subsection-name": "subsection-container-sizing-col1"
+ },
+ {
+ "config": "yarn-env/yarn_cgroups_enabled",
+ "subsection-name": "subsection-container-sizing-col1"
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.resource.percentage-physical-cpu-limit",
+ "subsection-name": "subsection-container-sizing-col1"
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.resource.cpu-vcores",
+ "subsection-name": "subsection-container-sizing-col1"
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.minimum-allocation-vcores",
+ "subsection-name": "subsection-container-sizing-col2"
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.maximum-allocation-vcores",
+ "subsection-name": "subsection-container-sizing-col2"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "yarn-site/yarn.nodemanager.resource.memory-mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.resource.percentage-physical-cpu-limit",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "percent"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.nodemanager.resource.cpu-vcores",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.minimum-allocation-mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.maximum-allocation-mb",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.minimum-allocation-vcores",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.scheduler.maximum-allocation-vcores",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ },
+ {
+ "config": "yarn-site/yarn.node-labels.enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "yarn-env/yarn_cgroups_enabled",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "yarn-site/yarn.resourcemanager.scheduler.monitor.enable",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "capacity-scheduler/yarn.scheduler.capacity.resource-calculator",
+ "widget": {
+ "type": "toggle"
+ }
+ }
+ ]
+ }
+}
[39/50] [abbrv] ambari git commit: AMBARI-19093. HDP 3.0 support for
HDFS with configs, kerberos, widgets, metrics, quicklinks,
and themes (alejandro)
Posted by nc...@apache.org.
AMBARI-19093. HDP 3.0 support for HDFS with configs, kerberos, widgets, metrics, quicklinks, and themes (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a710b35
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a710b35
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a710b35
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 2a710b3587965c09e7b034ea79a7eca31fc35c25
Parents: aed9839
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Dec 5 15:00:07 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Tue Dec 6 16:53:31 2016 -0800
----------------------------------------------------------------------
.../services/HDFS/configuration/core-site.xml | 56 ++
.../services/HDFS/configuration/hadoop-env.xml | 200 ++++++
.../services/HDFS/configuration/hdfs-log4j.xml | 226 +++++++
.../services/HDFS/configuration/hdfs-site.xml | 153 +++++
.../HDFS/configuration/ranger-hdfs-audit.xml | 217 +++++++
.../ranger-hdfs-plugin-properties.xml | 98 +++
.../configuration/ranger-hdfs-policymgr-ssl.xml | 67 ++
.../HDFS/configuration/ranger-hdfs-security.xml | 65 ++
.../services/HDFS/configuration/widgets.json | 649 +++++++++++++++++++
.../stacks/HDP/3.0/services/HDFS/kerberos.json | 246 +++++++
.../stacks/HDP/3.0/services/HDFS/metainfo.xml | 190 ++++++
.../services/HDFS/quicklinks/quicklinks.json | 80 +++
.../HDP/3.0/services/HDFS/themes/theme.json | 179 +++++
13 files changed, 2426 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml
new file mode 100644
index 0000000..9dcf561
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!-- Put site-specific property overrides in this file. -->
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>hadoop.http.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>
+ Indicates if anonymous requests are allowed when using 'simple' authentication.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hadoop.security.key.provider.path</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_host</name>
+ </property>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_port</name>
+ </property>
+ <property>
+ <type>kms-env</type>
+ <name>kms_port</name>
+ </property>
+ <property>
+ <type>ranger-kms-site</type>
+ <name>ranger.service.https.attrib.ssl.enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml
new file mode 100644
index 0000000..8697740
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>keyserver_host</name>
+ <value> </value>
+ <display-name>Key Server Host</display-name>
+ <description>Hostnames where Key Management Server is installed</description>
+ <value-attributes>
+ <type>string</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>keyserver_port</name>
+ <value/>
+ <display-name>Key Server Port</display-name>
+ <description>Port number where Key Management Server is available</description>
+ <value-attributes>
+ <type>int</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.3 -->
+ <!-- hadoop-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>hadoop-env template</display-name>
+ <description>This is the jinja template for hadoop-env.sh file</description>
+ <value>
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME. All others are
+# optional. When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use. Required.
+export JAVA_HOME={{java_home}}
+export HADOOP_HOME_WARN_SUPPRESS=1
+
+# Hadoop home directory
+export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+# Hadoop Configuration Directory
+
+{# this is different for HDP1 #}
+# Path to jsvc required by secure HDP 2.0 datanode
+export JSVC_HOME={{jsvc_path}}
+
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+export HADOOP_HEAPSIZE="{{hadoop_heapsize}}"
+
+export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}"
+
+# Extra Java runtime options. Empty by default.
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
+
+# Command specific options appended to HADOOP_OPTS when specified
+HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
+
+HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
+
+{% if java_version < 8 %}
+SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT"
+export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}"
+export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS"
+
+{% else %}
+SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT"
+export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}"
+export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
+{% endif %}
+
+HADOOP_NFS3_OPTS="-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}"
+HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}"
+
+
+# On secure datanodes, user to run the datanode as after dropping privileges
+export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}
+
+# Extra ssh options. Empty by default.
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
+
+# Where log files are stored. $HADOOP_HOME/logs by default.
+export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER
+
+# History server logs
+export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER
+
+# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.
+# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
+
+# host:path where hadoop code should be rsync'd from. Unset by default.
+# export HADOOP_MASTER=master:/home/$USER/src/hadoop
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HADOOP_SLAVE_SLEEP=0.1
+
+# The directory where pid files are stored. /tmp by default.
+export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER
+export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER
+
+# History server pid
+export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER
+
+YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY"
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+
+# export HADOOP_NICENESS=10
+
+# Add database libraries
+JAVA_JDBC_LIBS=""
+if [ -d "/usr/share/java" ]; then
+ for jarFile in `ls /usr/share/java | grep -E "(mysql|ojdbc|postgresql|sqljdbc)" 2>/dev/null`
+ do
+ JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
+ done
+fi
+
+# Add libraries to the hadoop classpath - some may not need a colon as they already include it
+export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}
+
+# Setting path to hdfs command line
+export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
+
+# Mostly required for hadoop 2.0
+export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}
+
+export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS"
+
+
+# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin.
+# Makes sense to fix only when runing DN as root
+if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ {% if is_datanode_max_locked_memory_set %}
+ ulimit -l {{datanode_max_locked_memory}}
+ {% endif %}
+ ulimit -n {{hdfs_user_nofile_limit}}
+fi
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nfsgateway_heapsize</name>
+ <display-name>NFSGateway maximum Java heap size</display-name>
+ <value>1024</value>
+ <description>Maximum Java heap size for NFSGateway (Java option -Xmx)</description>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml
new file mode 100644
index 0000000..215a6ee
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml
@@ -0,0 +1,226 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>content</name>
+ <display-name>hdfs-log4j template</display-name>
+ <description>Custom log4j.properties</description>
+ <value>
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+# Define some default values that can be overridden by system properties
+# To change daemon root logger use hadoop_root_logger in hadoop-env
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+#Security audit appender
+#
+hadoop.security.logger=INFO,console
+hadoop.security.log.maxfilesize=256MB
+hadoop.security.log.maxbackupindex=20
+log4j.category.SecurityLogger=${hadoop.security.logger}
+hadoop.security.log.file=SecurityAuth.audit
+log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
+
+log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
+log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
+
+#
+# hdfs audit logging
+#
+hdfs.audit.logger=INFO,console
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
+log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
+log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
+log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd
+
+#
+# NameNode metrics logging.
+# The default is to retain two namenode-metrics.log files up to 64MB each.
+#
+namenode.metrics.logger=INFO,NullAppender
+log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}
+log4j.additivity.NameNodeMetricsLog=false
+log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender
+log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log
+log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n
+log4j.appender.NNMETRICSRFA.MaxBackupIndex=1
+log4j.appender.NNMETRICSRFA.MaxFileSize=64MB
+
+#
+# mapred audit logging
+#
+mapred.audit.logger=INFO,console
+log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
+log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
+log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
+log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
+
+#
+# Rolling File Appender
+#
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+log4j.appender.RFA.MaxFileSize=256MB
+log4j.appender.RFA.MaxBackupIndex=10
+
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+# Custom Logging levels
+
+hadoop.metrics.log.level=INFO
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+# Removes "deprecated" messages
+log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
+
+#
+# HDFS block state change log from block manager
+#
+# Uncomment the following to suppress normal block state change
+# messages from BlockManager in NameNode.
+#log4j.logger.BlockStateChange=WARN
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml
new file mode 100644
index 0000000..ac141d1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml
@@ -0,0 +1,153 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration supports_final="true">
+ <!-- These configs were inherited from HDP 2.1 -->
+ <property>
+ <name>dfs.namenode.audit.log.async</name>
+ <value>true</value>
+ <description>Whether to enable async auditlog</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.namenode.fslock.fair</name>
+ <value>false</value>
+ <description>Whether fsLock is fair</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>dfs.namenode.startup.delay.block.deletion.sec</name>
+ <value>3600</value>
+ <description>
+ The delay in seconds at which we will pause the blocks deletion
+ after Namenode startup. By default it's disabled.
+ In the case a directory has large number of directories and files are
+ deleted, suggested delay is one hour to give the administrator enough time
+ to notice large number of pending deletion blocks and take corrective
+ action.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.journalnode.edits.dir</name>
+ <value>/hadoop/hdfs/journalnode</value>
+ <description>The path where the JournalNode daemon will store its local state. </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.client.retry.policy.enabled</name>
+ <value>false</value>
+ <description>Enables HDFS client retry in the event of a NameNode failure.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.content-summary.limit</name>
+ <value>5000</value>
+ <description>Dfs content summary limit.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.encryption.key.provider.uri</name>
+ <description>
+ The KeyProvider to use when interacting with encryption keys used
+ when reading and writing to an encryption zone.
+ </description>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_host</name>
+ </property>
+ <property>
+ <type>hadoop-env</type>
+ <name>keyserver_port</name>
+ </property>
+ <property>
+ <type>kms-env</type>
+ <name>kms_port</name>
+ </property>
+ <property>
+ <type>ranger-kms-site</type>
+ <name>ranger.service.https.attrib.ssl.enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>nfs.file.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ <display-name>NFSGateway dump directory</display-name>
+ <description>
+ This directory is used to temporarily save out-of-order writes before
+ writing to HDFS. For each file, the out-of-order writes are dumped after
+ they are accumulated to exceed certain threshold (e.g., 1MB) in memory.
+ One needs to make sure the directory has enough space.
+ </description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ <description>
+ By default, the export can be mounted by any client. To better control the access,
+ users can update the following property. The value string contains machine name and access privilege,
+ separated by whitespace characters. Machine name format can be single host, wildcards, and IPv4
+ networks.The access privilege uses rw or ro to specify readwrite or readonly access of the machines
+ to exports. If the access privilege is not provided, the default is read-only. Entries are separated
+ by ";". For example: "192.168.0.0/22 rw ; host*.example.com ; host1.test.org ro;".
+ </description>
+ <display-name>Allowed hosts</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.encrypt.data.transfer.cipher.suites</name>
+ <value>AES/CTR/NoPadding</value>
+ <description>
+ This value may be either undefined or AES/CTR/NoPadding. If defined, then
+ dfs.encrypt.data.transfer uses the specified cipher suite for data encryption.
+ If not defined, then only the algorithm specified in dfs.encrypt.data.transfer.algorithm
+ is used. By default, the property is not defined.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dfs.namenode.inode.attributes.provider.class</name>
+ <description>Enable ranger hdfs plugin</description>
+ <depends-on>
+ <property>
+ <type>ranger-hdfs-plugin-properties</type>
+ <name>ranger-hdfs-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml
new file mode 100644
index 0000000..fd41817
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ <description>Is Audit enabled?</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db</name>
+ <value>false</value>
+ <display-name>Audit to DB</display-name>
+ <description>Is Audit to DB enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.db</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.url</name>
+ <value>{{audit_jdbc_url}}</value>
+ <description>Audit DB JDBC URL</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.user</name>
+ <value>{{xa_audit_db_user}}</value>
+ <description>Audit DB JDBC User</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.password</name>
+ <value>crypted</value>
+ <property-type>PASSWORD</property-type>
+ <description>Audit DB JDBC Password</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.driver</name>
+ <value>{{jdbc_driver}}</value>
+ <description>Audit DB JDBC Driver</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.credential.provider.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>Credential file store</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.batch.filespool.dir</name>
+ <value>/var/log/hadoop/hdfs/audit/db/spool</value>
+ <description>/var/log/hadoop/hdfs/audit/db/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <value>true</value>
+ <display-name>Audit to HDFS</display-name>
+ <description>Is Audit to HDFS enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <value>hdfs://NAMENODE_HOSTNAME:8020/ranger/audit</value>
+ <description>HDFS folder to write audit to, make sure the service user has requried permissions</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
+ <value>/var/log/hadoop/hdfs/audit/hdfs/spool</value>
+ <description>/var/log/hadoop/hdfs/audit/hdfs/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr</name>
+ <value>false</value>
+ <display-name>Audit to SOLR</display-name>
+ <description>Is Solr audit enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.solr</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.urls</name>
+ <value/>
+ <description>Solr URL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.urls</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.zookeepers</name>
+ <value>NONE</value>
+ <description>Solr Zookeeper string</description>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.zookeepers</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.batch.filespool.dir</name>
+ <value>/var/log/hadoop/hdfs/audit/solr/spool</value>
+ <description>/var/log/hadoop/hdfs/audit/solr/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.provider.summary.enabled</name>
+ <value>false</value>
+ <display-name>Audit provider summary enabled</display-name>
+ <description>Enable Summary audit?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- These configs are deleted in HDP 2.5. -->
+ <property>
+ <name>xasecure.audit.destination.db</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.url</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.user</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.password</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.driver</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.credential.provider.file</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.batch.filespool.dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml
new file mode 100644
index 0000000..b31742c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>policy_user</name>
+ <value>ambari-qa</value>
+ <display-name>Policy user for HDFS</display-name>
+ <description>This user must be system user and also present at Ranger
+ admin portal</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value/>
+ <description>Used for repository creation on ranger admin
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>common.name.for.certificate</name>
+ <value/>
+ <description>Common name for certificate, this value should match what is specified in repo within ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger-hdfs-plugin-enabled</name>
+ <value>No</value>
+ <display-name>Enable Ranger for HDFS</display-name>
+ <description>Enable ranger hdfs plugin</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>ranger-hdfs-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_USERNAME</name>
+ <value>hadoop</value>
+ <display-name>Ranger repository config user</display-name>
+ <description>Used for repository creation on ranger admin
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_PASSWORD</name>
+ <value>hadoop</value>
+ <display-name>Ranger repository config password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Used for repository creation on ranger admin
+ </description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.5 -->
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ <description>Used for repository creation on ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false" />
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml
new file mode 100644
index 0000000..1bc83df
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks</value>
+ <description>Java Keystore files</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.password</name>
+ <value>myKeyFilePassword</value>
+ <property-type>PASSWORD</property-type>
+ <description>password for keystore</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks</value>
+ <description>java truststore file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.password</name>
+ <value>changeit</value>
+ <property-type>PASSWORD</property-type>
+ <description>java truststore password</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java keystore credential file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java truststore credential file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml
new file mode 100644
index 0000000..1b0a821
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>ranger.plugin.hdfs.service.name</name>
+ <value>{{repo_name}}</value>
+ <description>Name of the Ranger service containing Hdfs policies</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.hdfs.policy.source.impl</name>
+ <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+ <description>Class to retrieve policies from the source</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.hdfs.policy.rest.url</name>
+ <value>{{policymgr_mgr_url}}</value>
+ <description>URL to Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.hdfs.policy.rest.ssl.config.file</name>
+ <value>/etc/hadoop/conf/ranger-policymgr-ssl.xml</value>
+ <description>Path to the file containing SSL details to contact Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.hdfs.policy.pollIntervalMs</name>
+ <value>30000</value>
+ <description>How often to poll for changes in policies?</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.hdfs.policy.cache.dir</name>
+ <value>/etc/ranger/{{repo_name}}/policycache</value>
+ <description>Directory where Ranger policies are cached after successful retrieval from the source</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.add-hadoop-authorization</name>
+ <value>true</value>
+ <description>Enable/Disable the default hadoop authorization (based on rwxrwxrwx permission on the resource) if Ranger Authorization fails.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json
new file mode 100644
index 0000000..4a645b0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json
@@ -0,0 +1,649 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_hdfs_dashboard",
+ "display_name": "Standard HDFS Dashboard",
+ "section_name": "HDFS_SUMMARY",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "NameNode GC count",
+ "description": "Count of total garbage collections and count of major type garbage collections of the JVM.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.JvmMetrics.GcCount._rate",
+ "metric_path": "metrics/jvm/gcCount._rate",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate",
+ "metric_path": "metrics/jvm/GcCountConcurrentMarkSweep._rate",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "GC total count",
+ "value": "${jvm.JvmMetrics.GcCount._rate}"
+ },
+ {
+ "name": "GC count of type major collection",
+ "value": "${jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NameNode GC time",
+ "description": "Total time taken by major type garbage collections in milliseconds.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate",
+ "metric_path": "metrics/jvm/GcTimeMillisConcurrentMarkSweep._rate",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "GC time in major collection",
+ "value": "${jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate}"
+ }
+ ],
+ "properties": {
+ "display_unit": "ms",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NN Connection Load",
+ "description": "Number of open RPC connections being managed by NameNode.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "rpc.rpc.client.NumOpenConnections",
+ "metric_path": "metrics/rpc/client/NumOpenConnections",
+ "category": "",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "rpc.rpc.datanode.NumOpenConnections",
+ "metric_path": "metrics/rpc/datanode/NumOpenConnections",
+ "category": "",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "Open Client Connections",
+ "value": "${rpc.rpc.client.NumOpenConnections}"
+ },
+ {
+ "name": "Open Datanode Connections",
+ "value": "${rpc.rpc.datanode.NumOpenConnections}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NameNode Heap",
+ "description": "Heap memory committed and Heap memory used with respect to time.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "jvm.JvmMetrics.MemHeapCommittedM",
+ "metric_path": "metrics/jvm/memHeapCommittedM",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "jvm.JvmMetrics.MemHeapUsedM",
+ "metric_path": "metrics/jvm/memHeapUsedM",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "JVM heap committed",
+ "value": "${jvm.JvmMetrics.MemHeapCommittedM}"
+ },
+ {
+ "name": "JVM heap used",
+ "value": "${jvm.JvmMetrics.MemHeapUsedM}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "NameNode Host Load",
+ "description": "Percentage of CPU and Memory resources being consumed on NameNode host.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "cpu_system",
+ "metric_path": "metrics/cpu/cpu_system",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_user",
+ "metric_path": "metrics/cpu/cpu_user",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_nice",
+ "metric_path": "metrics/cpu/cpu_nice",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_idle",
+ "metric_path": "metrics/cpu/cpu_idle",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "cpu_wio",
+ "metric_path": "metrics/cpu/cpu_wio",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "mem_total",
+ "metric_path": "metrics/memory/mem_total",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "mem_free",
+ "metric_path": "metrics/memory/mem_free",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "CPU utilization",
+ "value": "${((cpu_system + cpu_user + cpu_nice)/(cpu_system + cpu_user + cpu_nice + cpu_idle + cpu_wio)) * 100}"
+ },
+ {
+ "name": "Memory utilization",
+ "value": "${((mem_total - mem_free)/mem_total) * 100}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1",
+ "display_unit": "%"
+ }
+ },
+ {
+ "widget_name": "NameNode RPC",
+ "description": "Compares the average time spent for RPC request in a queue and RPC request being processed.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "rpc.rpc.client.RpcQueueTimeAvgTime",
+ "metric_path": "metrics/rpc/client/RpcQueueTime_avg_time",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "rpc.rpc.client.RpcProcessingTimeAvgTime",
+ "metric_path": "metrics/rpc/client/RpcProcessingTime_avg_time",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "rpc.rpc.datanode.RpcQueueTimeAvgTime",
+ "metric_path": "metrics/rpc/datanode/RpcQueueTime_avg_time",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ },
+ {
+ "name": "rpc.rpc.datanode.RpcProcessingTimeAvgTime",
+ "metric_path": "metrics/rpc/datanode/RpcProcessingTime_avg_time",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "Client RPC Queue Wait time",
+ "value": "${rpc.rpc.client.RpcQueueTimeAvgTime}"
+ },
+ {
+ "name": "Client RPC Processing time",
+ "value": "${rpc.rpc.client.RpcProcessingTimeAvgTime}"
+ },
+ {
+ "name": "Datanode RPC Queue Wait time",
+ "value": "${rpc.rpc.datanode.RpcQueueTimeAvgTime}"
+ },
+ {
+ "name": "Datanode RPC Processing time",
+ "value": "${rpc.rpc.datanode.RpcProcessingTimeAvgTime}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1",
+ "display_unit": "ms"
+ }
+ },
+ {
+ "widget_name": "NameNode Operations",
+ "description": "Rate per second of number of file operation over time.",
+ "widget_type": "GRAPH",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "dfs.namenode.TotalFileOps._rate",
+ "metric_path": "metrics/dfs/namenode/TotalFileOps._rate",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "NameNode File Operations",
+ "value": "${dfs.namenode.TotalFileOps._rate}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Failed disk volumes",
+ "description": "Number of Failed disk volumes across all DataNodes. Its indicative of HDFS bad health.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes._sum",
+ "metric_path": "metrics/dfs/datanode/NumFailedVolumes",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "Failed disk volumes",
+ "value": "${FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes._sum}"
+ }
+ ],
+ "properties": {
+ "display_unit": ""
+ }
+ },
+ {
+ "widget_name": "Blocks With Corrupted Replicas",
+ "description": "Number represents data blocks with at least one corrupted replica (but not all of them). Its indicative of HDFS bad health.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "Hadoop:service=NameNode,name=FSNamesystem.CorruptBlocks",
+ "metric_path": "metrics/dfs/FSNamesystem/CorruptBlocks",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "Blocks With Corrupted Replicas",
+ "value": "${Hadoop:service=NameNode,name=FSNamesystem.CorruptBlocks}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0",
+ "error_threshold": "50"
+ }
+ },
+ {
+ "widget_name": "Under Replicated Blocks",
+ "description": "Number represents file blocks that does not meet the replication factor criteria. Its indicative of HDFS bad health.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks",
+ "metric_path": "metrics/dfs/FSNamesystem/UnderReplicatedBlocks",
+ "service_name": "HDFS",
+ "component_name": "NAMENODE",
+ "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active"
+ }
+ ],
+ "values": [
+ {
+ "name": "Under Replicated Blocks",
+ "value": "${Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0",
+ "error_threshold": "50"
+ }
+ },
+ {
+ "widget_name": "HDFS Space Utilization",
+ "description": "Percentage of available space used in the DFS.",
+ "widget_type": "GAUGE",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining",
+ "metric_path": "metrics/FSDatasetState/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl/Remaining",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity",
+ "metric_path": "metrics/dfs/datanode/Capacity",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "HDFS Space Utilization",
+ "value": "${(FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity - FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining)/FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity}"
+ }
+ ],
+ "properties": {
+ "warning_threshold": "0.75",
+ "error_threshold": "0.9"
+ }
+ }
+ ]
+ },
+ {
+ "layout_name": "default_hdfs_heatmap",
+ "section_name": "HDFS_HEATMAPS",
+ "display_name": "HDFS Heatmaps",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "HDFS Bytes Read",
+ "default_section_name": "HDFS_HEATMAPS",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "dfs.datanode.BytesRead._rate",
+ "metric_path": "metrics/dfs/datanode/bytes_read._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "HDFS Bytes Read",
+ "value": "${dfs.datanode.BytesRead._rate}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "max_limit": "1024"
+ }
+ },
+ {
+ "widget_name": "HDFS Bytes Written",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "dfs.datanode.BytesWritten._rate",
+ "metric_path": "metrics/dfs/datanode/bytes_written._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "HDFS Bytes Written",
+ "value": "${dfs.datanode.BytesWritten._rate}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "max_limit": "1024"
+ }
+ },
+ {
+ "widget_name": "DataNode Garbage Collection Time",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "Hadoop:service=DataNode,name=JvmMetrics.GcTimeMillis",
+ "metric_path": "metrics/jvm/gcTimeMillis",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "DataNode Garbage Collection Time",
+ "value": "${Hadoop:service=DataNode,name=JvmMetrics.GcTimeMillis}"
+ }
+ ],
+ "properties": {
+ "display_unit": "ms",
+ "max_limit": "10000"
+ }
+ },
+ {
+ "widget_name": "DataNode JVM Heap Memory Used",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "Hadoop:service=DataNode,name=JvmMetrics.MemHeapUsedM",
+ "metric_path": "metrics/jvm/memHeapUsedM",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "DataNode JVM Heap Memory Used",
+ "value": "${Hadoop:service=DataNode,name=JvmMetrics.MemHeapUsedM}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "max_limit": "512"
+ }
+ },
+ {
+ "widget_name": "DataNode JVM Heap Memory Committed",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "Hadoop:service=DataNode,name=JvmMetrics.MemHeapCommittedM",
+ "metric_path": "metrics/jvm/memHeapCommittedM",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "DataNode JVM Heap Memory Committed",
+ "value": "${Hadoop:service=DataNode,name=JvmMetrics.MemHeapCommittedM}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "max_limit": "512"
+ }
+ },
+ {
+ "widget_name": "DataNode Process Disk I/O Utilization",
+ "default_section_name": "HDFS_HEATMAPS",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "dfs.datanode.BytesRead._rate",
+ "metric_path": "metrics/dfs/datanode/bytes_read._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "dfs.datanode.BytesWritten._rate",
+ "metric_path": "metrics/dfs/datanode/bytes_written._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "dfs.datanode.TotalReadTime._rate",
+ "metric_path": "metrics/dfs/datanode/TotalReadTime._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "dfs.datanode.TotalWriteTime._rate",
+ "metric_path": "metrics/dfs/datanode/TotalWriteTime._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "DataNode Process Disk I/O Utilization",
+ "value": "${((dfs.datanode.BytesRead._rate/dfs.datanode.TotalReadTime._rate)+(dfs.datanode.BytesWritten._rate/dfs.datanode.TotalWriteTime._rate))*50}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "DataNode Process Network I/O Utilization",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "dfs.datanode.RemoteBytesRead._rate",
+ "metric_path": "metrics/dfs/datanode/RemoteBytesRead._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "dfs.datanode.ReadsFromRemoteClient._rate",
+ "metric_path": "metrics/dfs/datanode/reads_from_remote_client._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "dfs.datanode.RemoteBytesWritten._rate",
+ "metric_path": "metrics/dfs/datanode/RemoteBytesWritten._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "dfs.datanode.WritesFromRemoteClient._rate",
+ "metric_path": "metrics/dfs/datanode/writes_from_remote_client._rate",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "DataNode Process Network I/O Utilization",
+ "value": "${((dfs.datanode.RemoteBytesRead._rate/dfs.datanode.ReadsFromRemoteClient._rate)+(dfs.datanode.RemoteBytesWritten._rate/dfs.datanode.WritesFromRemoteClient._rate))*50}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "HDFS Space Utilization",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining",
+ "metric_path": "metrics/FSDatasetState/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl/Remaining",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ },
+ {
+ "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity",
+ "metric_path": "metrics/dfs/datanode/Capacity",
+ "service_name": "HDFS",
+ "component_name": "DATANODE"
+ }
+ ],
+ "values": [
+ {
+ "name": "HDFS Space Utilization",
+ "value": "${((FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity - FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining)/FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json
new file mode 100644
index 0000000..9000e95
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json
@@ -0,0 +1,246 @@
+{
+ "services": [
+ {
+ "name": "HDFS",
+ "identities": [
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "hdfs-site/dfs.web.authentication.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "hdfs-site/dfs.web.authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "auth_to_local_properties" : [
+ "core-site/hadoop.security.auth_to_local"
+ ],
+ "configurations": [
+ {
+ "core-site": {
+ "hadoop.security.authentication": "kerberos",
+ "hadoop.security.authorization": "true",
+ "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}"
+ }
+ },
+ {
+ "ranger-hdfs-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "HDFS_CLIENT",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ }
+ ]
+ },
+ {
+ "name": "NAMENODE",
+ "identities": [
+ {
+ "name": "hdfs",
+ "principal": {
+ "value": "${hadoop-env/hdfs_user}-${cluster_name|toLower()}@${realm}",
+ "type" : "user" ,
+ "configuration": "hadoop-env/hdfs_principal_name",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hdfs.headless.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hadoop-env/hdfs_user_keytab"
+ }
+ },
+ {
+ "name": "namenode_nn",
+ "principal": {
+ "value": "nn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hdfs-site/dfs.namenode.kerberos.principal",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/nn.service.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hdfs-site/dfs.namenode.keytab.file"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "hdfs-site/dfs.namenode.kerberos.internal.spnego.principal"
+ }
+ },
+ {
+ "name": "/HDFS/NAMENODE/namenode_nn",
+ "principal": {
+ "configuration": "ranger-hdfs-audit/xasecure.audit.jaas.Client.option.principal"
+ },
+ "keytab": {
+ "configuration": "ranger-hdfs-audit/xasecure.audit.jaas.Client.option.keyTab"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "hdfs-site": {
+ "dfs.block.access.token.enable": "true"
+ }
+ }
+ ]
+ },
+ {
+ "name": "DATANODE",
+ "identities": [
+ {
+ "name": "datanode_dn",
+ "principal": {
+ "value": "dn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hdfs-site/dfs.datanode.kerberos.principal",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/dn.service.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hdfs-site/dfs.datanode.keytab.file"
+ }
+ }
+ ],
+ "configurations" : [
+ {
+ "hdfs-site" : {
+ "dfs.datanode.address" : "0.0.0.0:1019",
+ "dfs.datanode.http.address": "0.0.0.0:1022"
+ }
+ }
+ ]
+ },
+ {
+ "name": "SECONDARY_NAMENODE",
+ "identities": [
+ {
+ "name": "secondary_namenode_nn",
+ "principal": {
+ "value": "nn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.principal",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/nn.service.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hdfs-site/dfs.secondary.namenode.keytab.file"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.internal.spnego.principal"
+ }
+ }
+ ]
+ },
+ {
+ "name": "NFS_GATEWAY",
+ "identities": [
+ {
+ "name": "nfsgateway",
+ "principal": {
+ "value": "nfs/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hdfs-site/nfs.kerberos.principal",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/nfs.service.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hdfs-site/nfs.keytab.file"
+ }
+ }
+ ]
+ },
+ {
+ "name": "JOURNALNODE",
+ "identities": [
+ {
+ "name": "journalnode_jn",
+ "principal": {
+ "value": "jn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hdfs-site/dfs.journalnode.kerberos.principal",
+ "local_username" : "${hadoop-env/hdfs_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/jn.service.keytab",
+ "owner": {
+ "name": "${hadoop-env/hdfs_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hdfs-site/dfs.journalnode.keytab.file"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "hdfs-site/dfs.journalnode.kerberos.internal.spnego.principal"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
[43/50] [abbrv] ambari git commit: AMBARI-18966. Add check to ensure
we do not have @Transactional annotations on private methods. (Attila
Doroszlai via stoader)
Posted by nc...@apache.org.
AMBARI-18966. Add check to ensure we do not have @Transactional annotations on private methods. (Attila Doroszlai via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9170b499
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9170b499
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9170b499
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 9170b49933106a0af651530d827af4f4e73f1541
Parents: 1e261db
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Wed Dec 7 09:52:10 2016 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Wed Dec 7 09:55:26 2016 +0100
----------------------------------------------------------------------
ambari-server/checkstyle.xml | 17 +++++++++++++++++
ambari-server/pom.xml | 4 ++++
.../internal/AlertTargetResourceProvider.java | 2 +-
.../ClusterStackVersionResourceProvider.java | 4 ++--
.../internal/UpgradeResourceProvider.java | 2 +-
.../apache/ambari/server/orm/dao/AlertsDAO.java | 6 +++---
.../apache/ambari/server/state/ServiceImpl.java | 2 +-
.../ambari/server/state/cluster/ClustersImpl.java | 2 +-
.../apache/ambari/server/state/host/HostImpl.java | 2 +-
.../state/services/RetryUpgradeActionService.java | 2 +-
.../svccomphost/ServiceComponentHostImpl.java | 2 +-
.../ambari/server/upgrade/UpgradeCatalog240.java | 4 ++--
12 files changed, 35 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/checkstyle.xml
----------------------------------------------------------------------
diff --git a/ambari-server/checkstyle.xml b/ambari-server/checkstyle.xml
new file mode 100644
index 0000000..81f6380
--- /dev/null
+++ b/ambari-server/checkstyle.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<!-- Licensed under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may obtain
+ a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+ required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ See accompanying LICENSE file. -->
+<!DOCTYPE module PUBLIC
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+ <module name="TreeWalker">
+ <module name="AvoidTransactionalOnPrivateMethodsCheck"/>
+ </module>
+</module>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 69ab9d0..e8de84e 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -692,6 +692,10 @@
<artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
</plugins>
<resources>
<resource>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
index 8e5674e..4010528 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
@@ -374,7 +374,7 @@ public class AlertTargetResourceProvider extends
*/
@Transactional
@SuppressWarnings("unchecked")
- private void updateAlertTargets(long alertTargetId,
+ void updateAlertTargets(long alertTargetId,
Map<String, Object> requestMap)
throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index 42aae84..d417ec2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -442,7 +442,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- private void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
+ void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
String desiredRepoVersion, RepositoryVersionState repoState)
throws AmbariException, SystemException {
final String clusterName = cluster.getClusterName();
@@ -482,7 +482,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- private RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
+ RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
List<Host> hosts, RepositoryVersionEntity repoVersionEnt, Map<String, Object> propertyMap)
throws AmbariException, SystemException {
final AmbariManagementController managementController = getManagementController();
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 5931743..67be152 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -972,7 +972,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
@Transactional
- private UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
+ UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
RequestStageContainer request,
UpgradeEntity upgradeEntity) throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
index d187947..e8c4b5f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
@@ -1492,7 +1492,7 @@ public class AlertsDAO implements Cleanable {
* @return a long representing the number of affected (deleted) records
*/
@Transactional
- private int cleanAlertNoticesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
+ int cleanAlertNoticesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
LOG.info("Deleting AlertNotice entities before date " + new Date(beforeDateMillis));
EntityManager entityManager = m_entityManagerProvider.get();
List<Integer> ids = findAllAlertHistoryIdsBeforeDate(clusterId, beforeDateMillis);
@@ -1523,7 +1523,7 @@ public class AlertsDAO implements Cleanable {
* @return a long representing the number of affected (deleted) records
*/
@Transactional
- private int cleanAlertCurrentsForClusterBeforeDate(long clusterId, long beforeDateMillis) {
+ int cleanAlertCurrentsForClusterBeforeDate(long clusterId, long beforeDateMillis) {
LOG.info("Deleting AlertCurrent entities before date " + new Date(beforeDateMillis));
EntityManager entityManager = m_entityManagerProvider.get();
List<Integer> ids = findAllAlertHistoryIdsBeforeDate(clusterId, beforeDateMillis);
@@ -1553,7 +1553,7 @@ public class AlertsDAO implements Cleanable {
*/
@Transactional
- private int cleanAlertHistoriesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
+ int cleanAlertHistoriesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
return executeQuery("AlertHistoryEntity.removeInClusterBeforeDate", AlertHistoryEntity.class, clusterId, beforeDateMillis);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
index f87b99c..e223eed 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
@@ -451,7 +451,7 @@ public class ServiceImpl implements Service {
}
@Transactional
- private void persistEntities(ClusterServiceEntity serviceEntity) {
+ void persistEntities(ClusterServiceEntity serviceEntity) {
long clusterId = cluster.getClusterId();
ClusterEntity clusterEntity = clusterDAO.findById(clusterId);
serviceEntity.setClusterEntity(clusterEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 8e5f37e..1de10f9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@ -183,7 +183,7 @@ public class ClustersImpl implements Clusters {
*/
@Inject
@Transactional
- private void loadClustersAndHosts() {
+ void loadClustersAndHosts() {
List<HostEntity> hostEntities = hostDAO.findAll();
for (HostEntity hostEntity : hostEntities) {
Host host = hostFactory.create(hostEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index dd5e635..a444f5d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@ -954,7 +954,7 @@ public class HostImpl implements Host {
}
@Transactional
- private void persistEntities(HostEntity hostEntity) {
+ void persistEntities(HostEntity hostEntity) {
hostDAO.create(hostEntity);
if (!hostEntity.getClusterEntities().isEmpty()) {
for (ClusterEntity clusterEntity : hostEntity.getClusterEntities()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
index 1ea5558..a92aa04 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
@@ -190,7 +190,7 @@ public class RetryUpgradeActionService extends AbstractScheduledService {
* @param requestId Request Id to search tasks for.
*/
@Transactional
- private void retryHoldingCommandsInRequest(Long requestId) {
+ void retryHoldingCommandsInRequest(Long requestId) {
if (requestId == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index c1655aa..5225598 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -1313,7 +1313,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
}
@Transactional
- private void persistEntities(HostEntity hostEntity, HostComponentStateEntity stateEntity,
+ void persistEntities(HostEntity hostEntity, HostComponentStateEntity stateEntity,
HostComponentDesiredStateEntity desiredStateEntity) {
ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName(
serviceComponent.getClusterId(), serviceComponent.getServiceName(),
http://git-wip-us.apache.org/repos/asf/ambari/blob/9170b499/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index c113fff..b7e248a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -1563,7 +1563,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
* @throws SQLException
*/
@Transactional
- private void updateServiceComponentDesiredStateTableDDL() throws SQLException {
+ void updateServiceComponentDesiredStateTableDDL() throws SQLException {
if (dbAccessor.tableHasPrimaryKey(SERVICE_COMPONENT_DS_TABLE, ID)) {
LOG.info("Skipping {} table Primary Key modifications since the new {} column already exists",
SERVICE_COMPONENT_DS_TABLE, ID);
@@ -2785,7 +2785,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
* instead of cluster_name
*/
@Transactional
- private void updateViewInstanceTable() throws SQLException {
+ void updateViewInstanceTable() throws SQLException {
try {
if (Long.class.equals(dbAccessor.getColumnClass(VIEWINSTANCE_TABLE, CLUSTER_HANDLE_COLUMN))) {
LOG.info(String.format("%s column is already numeric. Skipping an update of %s table.", CLUSTER_HANDLE_COLUMN, VIEWINSTANCE_TABLE));
[21/50] [abbrv] ambari git commit: AMBARI-19086. LDAP sync creates
groups with Local type (Attila Doroszla via rlevas)
Posted by nc...@apache.org.
AMBARI-19086. LDAP sync creates groups with Local type (Attila Doroszla via rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a00ffffc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a00ffffc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a00ffffc
Branch: refs/heads/branch-dev-patch-upgrade
Commit: a00ffffc54a8063c1728d2f3ba24db04504864da
Parents: b5122e2
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Mon Dec 5 16:42:14 2016 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Mon Dec 5 16:42:14 2016 -0500
----------------------------------------------------------------------
.../apache/ambari/server/orm/entities/GroupEntity.java | 5 +++--
.../ambari/server/security/authorization/Users.java | 10 ++++------
.../server/security/ldap/AmbariLdapDataPopulatorTest.java | 3 ++-
3 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a00ffffc/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/GroupEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/GroupEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/GroupEntity.java
index 58b2e5d..dc71b61 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/GroupEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/GroupEntity.java
@@ -101,7 +101,7 @@ public class GroupEntity {
return ldapGroup == 0 ? Boolean.FALSE : Boolean.TRUE;
}
- public void setLdapGroup(Boolean ldapGroup) {
+ private void setLdapGroup(Boolean ldapGroup) {
if (ldapGroup == null) {
this.ldapGroup = null;
} else {
@@ -113,8 +113,9 @@ public class GroupEntity {
return groupType;
}
- public void setgroupType(GroupType groupType) {
+ public void setGroupType(GroupType groupType) {
this.groupType = groupType;
+ setLdapGroup(groupType == GroupType.LDAP);
}
public Set<MemberEntity> getMemberEntities() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a00ffffc/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java
index 2cd538c..e69bbc9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/security/authorization/Users.java
@@ -49,10 +49,8 @@ import org.apache.ambari.server.orm.entities.PermissionEntity;
import org.apache.ambari.server.orm.entities.PrincipalEntity;
import org.apache.ambari.server.orm.entities.PrincipalTypeEntity;
import org.apache.ambari.server.orm.entities.PrivilegeEntity;
-import org.apache.ambari.server.orm.entities.ResourceEntity;
import org.apache.ambari.server.orm.entities.ResourceTypeEntity;
import org.apache.ambari.server.orm.entities.UserEntity;
-import org.apache.ambari.server.security.ClientSecurityType;
import org.apache.ambari.server.security.ldap.LdapBatchDto;
import org.apache.ambari.server.security.ldap.LdapUserGroupMemberDto;
import org.apache.commons.lang.StringUtils;
@@ -277,7 +275,7 @@ public class Users {
public synchronized void setGroupLdap(String groupName) throws AmbariException {
GroupEntity groupEntity = groupDAO.findGroupByName(groupName);
if (groupEntity != null) {
- groupEntity.setLdapGroup(true);
+ groupEntity.setGroupType(GroupType.LDAP);
groupDAO.merge(groupEntity);
} else {
throw new AmbariException("Group " + groupName + " doesn't exist");
@@ -435,7 +433,7 @@ public class Users {
final GroupEntity groupEntity = new GroupEntity();
groupEntity.setGroupName(groupName);
groupEntity.setPrincipal(principalEntity);
- groupEntity.setgroupType(groupType);
+ groupEntity.setGroupType(groupType);
groupDAO.create(groupEntity);
}
@@ -701,7 +699,7 @@ public class Users {
final Set<GroupEntity> groupsToBecomeLdap = new HashSet<GroupEntity>();
for (String groupName : batchInfo.getGroupsToBecomeLdap()) {
final GroupEntity groupEntity = groupDAO.findGroupByName(groupName);
- groupEntity.setLdapGroup(true);
+ groupEntity.setGroupType(GroupType.LDAP);
allGroups.put(groupEntity.getGroupName(), groupEntity);
groupsToBecomeLdap.add(groupEntity);
}
@@ -737,7 +735,7 @@ public class Users {
final GroupEntity groupEntity = new GroupEntity();
groupEntity.setGroupName(groupName);
groupEntity.setPrincipal(principalEntity);
- groupEntity.setLdapGroup(true);
+ groupEntity.setGroupType(GroupType.LDAP);
allGroups.put(groupEntity.getGroupName(), groupEntity);
groupsToCreate.add(groupEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a00ffffc/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
index 1866b12..2840e3d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java
@@ -38,6 +38,7 @@ import org.apache.ambari.server.orm.entities.PrivilegeEntity;
import org.apache.ambari.server.orm.entities.UserEntity;
import org.apache.ambari.server.security.authorization.AmbariLdapUtils;
import org.apache.ambari.server.security.authorization.Group;
+import org.apache.ambari.server.security.authorization.GroupType;
import org.apache.ambari.server.security.authorization.LdapServerProperties;
import org.apache.ambari.server.security.authorization.User;
import org.apache.ambari.server.security.authorization.Users;
@@ -1576,7 +1577,7 @@ public class AmbariLdapDataPopulatorTest {
final GroupEntity ldapGroup = new GroupEntity();
ldapGroup.setGroupId(1);
ldapGroup.setGroupName("ldapGroup");
- ldapGroup.setLdapGroup(true);
+ ldapGroup.setGroupType(GroupType.LDAP);
ldapGroup.setMemberEntities(new HashSet<MemberEntity>());
final User ldapUserWithoutGroup = createLdapUserWithoutGroup();
[14/50] [abbrv] ambari git commit: AMBARI-18065. Change Ranger Admin
start script to setup db for ranger (magyari_sandor)
Posted by nc...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/f5051dcb/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
new file mode 100644
index 0000000..030b717
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
@@ -0,0 +1,773 @@
+{
+ "localComponents": [
+ "SECONDARY_NAMENODE",
+ "HDFS_CLIENT",
+ "DATANODE",
+ "NAMENODE",
+ "RANGER_ADMIN",
+ "RANGER_TAGSYNC",
+ "RANGER_USERSYNC",
+ "ZOOKEEPER_SERVER",
+ "ZOOKEEPER_CLIENT",
+ "KERBEROS_CLIENT",
+ "LOGSEARCH_SOLR",
+ "LOGSEARCH_SOLR_CLIENT"
+ ],
+ "configuration_attributes": {
+ "ranger-hdfs-audit": {},
+ "ssl-client": {},
+ "ranger-admin-site": {},
+ "ranger-hdfs-policymgr-ssl": {},
+ "tagsync-application-properties": {},
+ "ranger-env": {},
+ "usersync-log4j": {},
+ "ranger-hdfs-plugin-properties": {},
+ "kerberos-env": {},
+ "admin-properties": {},
+ "ranger-ugsync-site": {},
+ "hdfs-site": {
+ "final": {
+ "dfs.datanode.data.dir": "true",
+ "dfs.namenode.http-address": "true",
+ "dfs.datanode.failed.volumes.tolerated": "true",
+ "dfs.support.append": "true",
+ "dfs.namenode.name.dir": "true",
+ "dfs.webhdfs.enabled": "true"
+ }
+ },
+ "ranger-tagsync-site": {},
+ "zoo.cfg": {},
+ "hadoop-policy": {},
+ "hdfs-log4j": {},
+ "krb5-conf": {},
+ "core-site": {
+ "final": {
+ "fs.defaultFS": "true"
+ }
+ },
+ "hadoop-env": {},
+ "zookeeper-log4j": {},
+ "ssl-server": {},
+ "ranger-site": {},
+ "admin-log4j": {},
+ "tagsync-log4j": {},
+ "ranger-hdfs-security": {},
+ "usersync-properties": {},
+ "zookeeper-env": {},
+ "infra-solr-env": {},
+ "infra-solr-client-log4j": {},
+ "cluster-env": {}
+ },
+ "public_hostname": "c6401.ambari.apache.org",
+ "commandId": "41-2",
+ "hostname": "c6401.ambari.apache.org",
+ "kerberosCommandParams": [],
+ "serviceName": "RANGER",
+ "role": "RANGER_ADMIN",
+ "forceRefreshConfigTagsBeforeExecution": [],
+ "requestId": 41,
+ "agentConfigParams": {
+ "agent": {
+ "parallel_execution": 0
+ }
+ },
+ "clusterName": "test_Cluster01",
+ "commandType": "EXECUTION_COMMAND",
+ "taskId": 186,
+ "roleParams": {},
+ "configurationTags": {
+ "ranger-hdfs-audit": {
+ "tag": "version1466705299922"
+ },
+ "ssl-client": {
+ "tag": "version1"
+ },
+ "ranger-admin-site": {
+ "tag": "version1467016680635"
+ },
+ "ranger-hdfs-policymgr-ssl": {
+ "tag": "version1466705299922"
+ },
+ "tagsync-application-properties": {
+ "tag": "version1467016680511"
+ },
+ "ranger-env": {
+ "tag": "version1466705299949"
+ },
+ "ranger-ugsync-site": {
+ "tag": "version1467016680537"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "tag": "version1466705299922"
+ },
+ "kerberos-env": {
+ "tag": "version1467016537243"
+ },
+ "admin-properties": {
+ "tag": "version1466705299949"
+ },
+ "hdfs-site": {
+ "tag": "version1467016680401"
+ },
+ "ranger-tagsync-site": {
+ "tag": "version1467016680586"
+ },
+ "zoo.cfg": {
+ "tag": "version1"
+ },
+ "hadoop-policy": {
+ "tag": "version1"
+ },
+ "hdfs-log4j": {
+ "tag": "version1"
+ },
+ "usersync-log4j": {
+ "tag": "version1466705299949"
+ },
+ "krb5-conf": {
+ "tag": "version1467016537243"
+ },
+ "core-site": {
+ "tag": "version1467016680612"
+ },
+ "hadoop-env": {
+ "tag": "version1467016680446"
+ },
+ "zookeeper-log4j": {
+ "tag": "version1"
+ },
+ "ssl-server": {
+ "tag": "version1"
+ },
+ "ranger-site": {
+ "tag": "version1466705299949"
+ },
+ "admin-log4j": {
+ "tag": "version1466705299949"
+ },
+ "tagsync-log4j": {
+ "tag": "version1466705299949"
+ },
+ "ranger-hdfs-security": {
+ "tag": "version1466705299922"
+ },
+ "usersync-properties": {
+ "tag": "version1466705299949"
+ },
+ "zookeeper-env": {
+ "tag": "version1467016680492"
+ },
+ "infra-solr-env": {
+ "tag": "version1467098537360"
+ },
+ "infra-solr-client-log4j": {
+ "tag": "version1467096917836"
+ },
+ "cluster-env": {
+ "tag": "version1467016680567"
+ }
+ },
+ "roleCommand": "START",
+ "hostLevelParams": {
+ "agent_stack_retry_on_unavailability": "false",
+ "stack_name": "HDP",
+ "package_version": "2_5_0_0_*",
+ "custom_mysql_jdbc_name": "mysql-connector-java.jar",
+ "previous_custom_mysql_jdbc_name": "mysql-connector-java-old.jar",
+ "host_sys_prepped": "false",
+ "ambari_db_rca_username": "mapred",
+ "current_version": "2.6.0.0-801",
+ "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar",
+ "agent_stack_retry_count": "5",
+ "stack_version": "2.6",
+ "jdk_name": "jdk-8u60-linux-x64.tar.gz",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "repository_version_id": "1",
+ "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
+ "not_managed_hdfs_path_list": "[\"/tmp\"]",
+ "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
+ "java_version": "8",
+ "repo_info": "[{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.5.0.0-801\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.5\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0\",\"latestBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.5.0.0-801\",\"baseSaved\":true},{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.21\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6\",\"baseSaved\":true}]",
+ "package_list": "[{\"name\":\"ranger_${stack_version}-admin\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-usersync\",\"condition\":\"\",\"skipUpgrade\":false},{\"name\":\"ranger_${stack_version}-tagsync\",\"condition\":\"should_install_ranger_tagsync\",\"skipUpgrade\":false},{\"name\":\"ambari-logsearch-solr-client\",\"condition\":\"should_install_logsearch_solr_client\",\"skipUpgrade\":false}]",
+ "db_name": "ambari",
+ "group_list": "[\"ranger\",\"hadoop\",\"users\"]",
+ "agentCacheDir": "/var/lib/ambari-agent/cache",
+ "ambari_db_rca_password": "mapred",
+ "jce_name": "jce_policy-8.zip",
+ "oracle_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//ojdbc6.jar",
+ "db_driver_filename": "mysql-connector-java.jar",
+ "user_list": "[\"zookeeper\",\"ambari-qa\",\"ranger\",\"hdfs\"]",
+ "clientsToUpdateConfigs": "[\"*\"]"
+ },
+ "commandParams": {
+ "service_package_folder": "common-services/RANGER/0.4.0/package",
+ "script": "scripts/ranger_admin.py",
+ "hooks_folder": "HDP/2.0.6/hooks",
+ "version": "2.6.0.0-801",
+ "max_duration_for_retries": "0",
+ "command_retry_enabled": "false",
+ "command_timeout": "600",
+ "script_type": "PYTHON"
+ },
+ "forceRefreshConfigTags": [],
+ "stageId": 2,
+ "clusterHostInfo": {
+ "snamenode_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "ambari_server_use_ssl": [
+ "false"
+ ],
+ "all_ping_ports": [
+ "8670"
+ ],
+ "ranger_tagsync_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "ranger_usersync_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "all_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "slave_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "namenode_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "ambari_server_port": [
+ "8080"
+ ],
+ "ranger_admin_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "all_racks": [
+ "/default-rack"
+ ],
+ "all_ipv4_ips": [
+ "172.22.83.73"
+ ],
+ "ambari_server_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "zookeeper_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "infra_solr_hosts": [
+ "c6401.ambari.apache.org"
+ ]
+ },
+ "configurations": {
+ "ranger-hdfs-audit": {
+ "xasecure.audit.destination.solr.zookeepers": "NONE",
+ "xasecure.audit.destination.solr.urls": "",
+ "xasecure.audit.destination.solr.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/solr/spool",
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/hdfs/spool",
+ "xasecure.audit.destination.hdfs": "true",
+ "xasecure.audit.destination.solr": "false",
+ "xasecure.audit.provider.summary.enabled": "false",
+ "xasecure.audit.destination.hdfs.dir": "hdfs://c6401.ambari.apache.org:8020/ranger/audit",
+ "xasecure.audit.is.enabled": "true"
+ },
+ "ssl-client": {
+ "ssl.client.truststore.reload.interval": "10000",
+ "ssl.client.keystore.password": "bigdata",
+ "ssl.client.truststore.type": "jks",
+ "ssl.client.keystore.location": "/etc/security/clientKeys/keystore.jks",
+ "ssl.client.truststore.location": "/etc/security/clientKeys/all.jks",
+ "ssl.client.truststore.password": "bigdata",
+ "ssl.client.keystore.type": "jks"
+ },
+ "ranger-admin-site": {
+ "ranger.is.solr.kerberised": "true",
+ "ranger.admin.kerberos.cookie.domain": "{{ranger_host}}",
+ "ranger.kms.service.user.hdfs": "hdfs",
+ "ranger.spnego.kerberos.principal": "HTTP/_HOST@EXAMPLE.COM",
+ "ranger.ldap.ad.url": "{{ranger_ug_ldap_url}}",
+ "ranger.plugins.hive.serviceuser": "hive",
+ "ranger.lookup.kerberos.keytab": "/etc/security/keytabs/rangerlookup.service.keytab",
+ "ranger.plugins.kms.serviceuser": "kms",
+ "ranger.service.https.attrib.ssl.enabled": "false",
+ "ranger.sso.browser.useragent": "Mozilla,chrome",
+ "ranger.jpa.jdbc.url": "jdbc:mysql://c6401.ambari.apache.org:3306/ranger01",
+ "ranger.plugins.hbase.serviceuser": "hbase",
+ "ranger.plugins.hdfs.serviceuser": "hdfs",
+ "ranger.ldap.group.searchbase": "{{ranger_ug_ldap_group_searchbase}}",
+ "ranger.ldap.user.dnpattern": "uid={0},ou=users,dc=xasecure,dc=net",
+ "ranger.plugins.knox.serviceuser": "knox",
+ "ranger.ldap.base.dn": "dc=example,dc=com",
+ "ranger.sso.publicKey": "",
+ "ranger.admin.kerberos.cookie.path": "/",
+ "ranger.service.https.attrib.clientAuth": "want",
+ "ranger.jpa.jdbc.user": "{{ranger_db_user}}",
+ "ranger.ldap.ad.user.searchfilter": "(sAMAccountName={0})",
+ "ranger.ldap.group.roleattribute": "cn",
+ "ranger.plugins.kafka.serviceuser": "kafka",
+ "ranger.admin.kerberos.principal": "rangeradmin/_HOST@EXAMPLE.COM",
+ "ranger.ldap.ad.bind.dn": "{{ranger_ug_ldap_bind_dn}}",
+ "ranger.credential.provider.path": "/etc/ranger/admin/rangeradmin.jceks",
+ "ranger.ldap.referral": "ignore",
+ "ranger.service.http.port": "6080",
+ "ranger.ldap.user.searchfilter": "(uid={0})",
+ "ranger.plugins.atlas.serviceuser": "atlas",
+ "ranger.truststore.password": "changeit",
+ "ranger.ldap.bind.password": "{{ranger_usersync_ldap_ldapbindpassword}}",
+ "ranger.audit.solr.password": "NONE",
+ "ranger.audit.solr.zookeepers": "c6401.ambari.apache.org:2181/ambari-solr",
+ "ranger.lookup.kerberos.principal": "rangerlookup/_HOST@EXAMPLE.COM",
+ "ranger.service.https.port": "6182",
+ "ranger.plugins.storm.serviceuser": "storm",
+ "ranger.externalurl": "{{ranger_external_url}}",
+ "ranger.truststore.file": "/etc/ranger/admin/conf/ranger-admin-keystore.jks",
+ "ranger.kms.service.user.hive": "",
+ "ranger.https.attrib.keystore.file": "/etc/ranger/admin/conf/ranger-admin-keystore.jks",
+ "ranger.jpa.jdbc.dialect": "{{jdbc_dialect}}",
+ "ranger.service.host": "{{ranger_host}}",
+ "ranger.service.https.attrib.keystore.keyalias": "rangeradmin",
+ "ranger.service.https.attrib.keystore.pass": "xasecure",
+ "ranger.unixauth.remote.login.enabled": "true",
+ "ranger.jpa.jdbc.credential.alias": "rangeradmin",
+ "ranger.ldap.ad.bind.password": "{{ranger_usersync_ldap_ldapbindpassword}}",
+ "ranger.audit.solr.username": "ranger_solr",
+ "ranger.sso.enabled": "false",
+ "ranger.audit.solr.urls": "",
+ "ranger.ldap.ad.domain": "",
+ "ranger.plugins.yarn.serviceuser": "yarn",
+ "ranger.audit.source.type": "solr",
+ "ranger.ldap.bind.dn": "{{ranger_ug_ldap_bind_dn}}",
+ "ranger.ldap.url": "{{ranger_ug_ldap_url}}",
+ "ranger.authentication.method": "UNIX",
+ "ranger.service.http.enabled": "true",
+ "ranger.ldap.group.searchfilter": "{{ranger_ug_ldap_group_searchfilter}}",
+ "ranger.ldap.ad.referral": "ignore",
+ "ranger.ldap.ad.base.dn": "dc=example,dc=com",
+ "ranger.jpa.jdbc.password": "_",
+ "ranger.spnego.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "ranger.sso.providerurl": "",
+ "ranger.unixauth.service.hostname": "{{ugsync_host}}",
+ "ranger.admin.kerberos.keytab": "/etc/security/keytabs/rangeradmin.service.keytab",
+ "ranger.admin.kerberos.token.valid.seconds": "30",
+ "ranger.jpa.jdbc.driver": "com.mysql.jdbc.Driver",
+ "ranger.unixauth.service.port": "5151"
+ },
+ "ranger-hdfs-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks",
+ "xasecure.policymgr.clientssl.truststore.password": "changeit",
+ "xasecure.policymgr.clientssl.keystore.credential.file": "jceks://file{{credential_file}}",
+ "xasecure.policymgr.clientssl.truststore": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks",
+ "xasecure.policymgr.clientssl.truststore.credential.file": "jceks://file{{credential_file}}",
+ "xasecure.policymgr.clientssl.keystore.password": "myKeyFilePassword"
+ },
+ "tagsync-application-properties": {
+ "atlas.kafka.zookeeper.connect": "c6401.ambari.apache.org:2181",
+ "atlas.kafka.security.protocol": "SASL_PLAINTEXT",
+ "atlas.jaas.KafkaClient.option.principal": "{{tagsync_jaas_principal}}",
+ "atlas.jaas.KafkaClient.option.keyTab": "{{tagsync_keytab_path}}",
+ "atlas.kafka.entities.group.id": "ranger_entities_consumer",
+ "atlas.jaas.KafkaClient.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "atlas.jaas.KafkaClient.option.serviceName": "kafka",
+ "atlas.kafka.bootstrap.servers": "localhost:6667",
+ "atlas.jaas.KafkaClient.option.useKeyTab": "true",
+ "atlas.jaas.KafkaClient.option.storeKey": "true",
+ "atlas.jaas.KafkaClient.loginModuleControlFlag": "required",
+ "atlas.kafka.sasl.kerberos.service.name": "kafka"
+ },
+ "ranger-env": {
+ "ranger_solr_shards": "1",
+ "ranger_solr_config_set": "ranger_audits",
+ "ranger_user": "ranger",
+ "ranger_solr_replication_factor": "1",
+ "xml_configurations_supported": "true",
+ "ranger-atlas-plugin-enabled": "No",
+ "ranger-hbase-plugin-enabled": "No",
+ "ranger-yarn-plugin-enabled": "No",
+ "bind_anonymous": "false",
+ "ranger_admin_username": "amb_ranger_admin",
+ "admin_password": "admin",
+ "is_solrCloud_enabled": "true",
+ "ranger-storm-plugin-enabled": "No",
+ "ranger-hdfs-plugin-enabled": "No",
+ "ranger_group": "ranger",
+ "ranger-knox-plugin-enabled": "No",
+ "ranger_admin_log_dir": "/var/log/ranger/admin",
+ "ranger-kafka-plugin-enabled": "No",
+ "ranger_privelege_user_jdbc_url": "jdbc:mysql://c6401.ambari.apache.org:3306",
+ "ranger-hive-plugin-enabled": "No",
+ "xasecure.audit.destination.solr": "true",
+ "ranger_pid_dir": "/var/run/ranger",
+ "xasecure.audit.destination.hdfs": "true",
+ "admin_username": "admin",
+ "xasecure.audit.destination.hdfs.dir": "hdfs://c6401.ambari.apache.org:8020/ranger/audit",
+ "create_db_dbuser": "true",
+ "ranger_solr_collection_name": "ranger_audits",
+ "ranger_admin_password": "P1!q9xa96SMi5NCl",
+ "ranger_usersync_log_dir": "/var/log/ranger/usersync"
+ },
+ "usersync-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nlog4j.rootLogger = info,logFile\n\n# logFile\nlog4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.logFile.file=${logdir}/usersync.
log\nlog4j.appender.logFile.datePattern='.'yyyy-MM-dd\nlog4j.appender.logFile.layout=org.apache.log4j.PatternLayout\nlog4j.appender.logFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n\n\n# console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.Target=System.out\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "hadoop.rpc.protection": "authentication",
+ "ranger-hdfs-plugin-enabled": "No",
+ "REPOSITORY_CONFIG_USERNAME": "hadoop",
+ "policy_user": "ambari-qa",
+ "common.name.for.certificate": "",
+ "REPOSITORY_CONFIG_PASSWORD": "hadoop"
+ },
+ "kerberos-env": {
+ "kdc_hosts": "c6401.ambari.apache.org",
+ "manage_auth_to_local": "true",
+ "install_packages": "true",
+ "realm": "EXAMPLE.COM",
+ "encryption_types": "aes des3-cbc-sha1 rc4 des-cbc-md5",
+ "ad_create_attributes_template": "\n{\n \"objectClass\": [\"top\", \"person\", \"organizationalPerson\", \"user\"],\n \"cn\": \"$principal_name\",\n #if( $is_service )\n \"servicePrincipalName\": \"$principal_name\",\n #end\n \"userPrincipalName\": \"$normalized_principal\",\n \"unicodePwd\": \"$password\",\n \"accountExpires\": \"0\",\n \"userAccountControl\": \"66048\"\n}",
+ "kdc_create_attributes": "",
+ "admin_server_host": "c6401.ambari.apache.org",
+ "group": "ambari-managed-principals",
+ "password_length": "20",
+ "ldap_url": "",
+ "manage_identities": "true",
+ "password_min_lowercase_letters": "1",
+ "create_ambari_principal": "true",
+ "service_check_principal_name": "${cluster_name|toLower()}-${short_date}",
+ "executable_search_paths": "/usr/bin, /usr/kerberos/bin, /usr/sbin, /usr/lib/mit/bin, /usr/lib/mit/sbin",
+ "password_chat_timeout": "5",
+ "kdc_type": "mit-kdc",
+ "set_password_expiry": "false",
+ "password_min_punctuation": "1",
+ "container_dn": "",
+ "case_insensitive_username_rules": "false",
+ "password_min_whitespace": "0",
+ "password_min_uppercase_letters": "1",
+ "password_min_digits": "1"
+ },
+ "admin-properties": {
+ "db_user": "rangeradmin01",
+ "DB_FLAVOR": "MYSQL",
+ "db_password": "rangeradmin01",
+ "db_root_user": "root",
+ "policymgr_external_url": "http://c6401.ambari.apache.org:6080",
+ "db_name": "ranger01",
+ "db_host": "c6401.ambari.apache.org",
+ "db_root_password": "vagrant",
+ "SQL_CONNECTOR_JAR": "{{driver_curl_target}}"
+ },
+ "ranger-ugsync-site": {
+ "ranger.usersync.ldap.binddn": "",
+ "ranger.usersync.policymgr.username": "rangerusersync",
+ "ranger.usersync.policymanager.mockrun": "false",
+ "ranger.usersync.group.searchbase": "",
+ "ranger.usersync.ldap.bindalias": "testldapalias",
+ "ranger.usersync.truststore.file": "/usr/hdp/current/ranger-usersync/conf/mytruststore.jks",
+ "ranger.usersync.port": "5151",
+ "ranger.usersync.pagedresultssize": "500",
+ "ranger.usersync.group.memberattributename": "",
+ "ranger.usersync.kerberos.principal": "rangerusersync/_HOST@EXAMPLE.COM",
+ "ranger.usersync.source.impl.class": "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder",
+ "ranger.usersync.ldap.referral": "ignore",
+ "ranger.usersync.group.searchfilter": "",
+ "ranger.usersync.ldap.user.objectclass": "person",
+ "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.ldap.user.searchfilter": "",
+ "ranger.usersync.ldap.groupname.caseconversion": "none",
+ "ranger.usersync.ldap.ldapbindpassword": "",
+ "ranger.usersync.unix.minUserId": "500",
+ "ranger.usersync.policymanager.maxrecordsperapicall": "1000",
+ "ranger.usersync.group.nameattribute": "",
+ "ranger.usersync.policymgr.alias": "ranger.usersync.policymgr.password",
+ "ranger.usersync.keystore.file": "/usr/hdp/current/ranger-usersync/conf/unixauthservice.jks",
+ "ranger.usersync.user.searchenabled": "false",
+ "ranger.usersync.group.usermapsyncenabled": "true",
+ "ranger.usersync.ldap.bindkeystore": "",
+ "ranger.usersync.ldap.user.groupnameattribute": "memberof, ismemberof",
+ "ranger.usersync.kerberos.keytab": "/etc/security/keytabs/rangerusersync.service.keytab",
+ "ranger.usersync.passwordvalidator.path": "./native/credValidator.uexe",
+ "ranger.usersync.group.objectclass": "",
+ "ranger.usersync.ldap.user.searchscope": "sub",
+ "ranger.usersync.unix.password.file": "/etc/passwd",
+ "ranger.usersync.ldap.user.nameattribute": "",
+ "ranger.usersync.pagedresultsenabled": "true",
+ "ranger.usersync.policymanager.baseURL": "{{ranger_external_url}}",
+ "ranger.usersync.group.search.first.enabled": "false",
+ "ranger.usersync.group.searchenabled": "false",
+ "ranger.usersync.sink.impl.class": "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder",
+ "ranger.usersync.ssl": "true",
+ "ranger.usersync.ldap.url": "",
+ "ranger.usersync.ldap.searchBase": "dc=hadoop,dc=apache,dc=org",
+ "ranger.usersync.policymgr.keystore": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks",
+ "ranger.usersync.ldap.user.searchbase": "",
+ "ranger.usersync.ldap.username.caseconversion": "none",
+ "ranger.usersync.credstore.filename": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks",
+ "ranger.usersync.keystore.password": "UnIx529p",
+ "ranger.usersync.unix.group.file": "/etc/group",
+ "ranger.usersync.filesource.file": "/tmp/usergroup.txt",
+ "ranger.usersync.group.searchscope": "",
+ "ranger.usersync.truststore.password": "changeit",
+ "ranger.usersync.enabled": "true",
+ "ranger.usersync.sleeptimeinmillisbetweensynccycle": "60000",
+ "ranger.usersync.filesource.text.delimiter": ","
+ },
+ "hdfs-site": {
+ "dfs.namenode.checkpoint.period": "21600",
+ "dfs.namenode.avoid.write.stale.datanode": "true",
+ "dfs.permissions.superusergroup": "hdfs",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.namenode.kerberos.internal.spnego.principal": "HTTP/_HOST@EXAMPLE.COM",
+ "dfs.datanode.kerberos.principal": "dn/_HOST@EXAMPLE.COM",
+ "dfs.heartbeat.interval": "3",
+ "dfs.content-summary.limit": "5000",
+ "dfs.support.append": "true",
+ "dfs.datanode.address": "0.0.0.0:1019",
+ "dfs.cluster.administrators": " hdfs",
+ "dfs.namenode.audit.log.async": "true",
+ "dfs.datanode.balance.bandwidthPerSec": "6250000",
+ "dfs.namenode.safemode.threshold-pct": "1",
+ "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
+ "dfs.namenode.rpc-address": "c6401.ambari.apache.org:8020",
+ "dfs.permissions.enabled": "true",
+ "dfs.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+ "dfs.client.read.shortcircuit": "true",
+ "dfs.https.port": "50470",
+ "dfs.namenode.https-address": "c6401.ambari.apache.org:50470",
+ "nfs.file.dump.dir": "/tmp/.hdfs-nfs",
+ "dfs.blocksize": "134217728",
+ "dfs.blockreport.initialDelay": "120",
+ "dfs.journalnode.edits.dir": "/hadoop/hdfs/journalnode",
+ "dfs.namenode.fslock.fair": "false",
+ "dfs.datanode.max.transfer.threads": "4096",
+ "dfs.secondary.namenode.kerberos.internal.spnego.principal": "HTTP/_HOST@EXAMPLE.COM",
+ "dfs.replication": "3",
+ "dfs.namenode.handler.count": "50",
+ "dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "fs.permissions.umask-mode": "022",
+ "dfs.namenode.stale.datanode.interval": "30000",
+ "dfs.datanode.ipc.address": "0.0.0.0:8010",
+ "dfs.datanode.failed.volumes.tolerated": "0",
+ "dfs.datanode.data.dir": "/grid/0/hadoop/hdfs/data",
+ "dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
+ "dfs.webhdfs.enabled": "true",
+ "dfs.encrypt.data.transfer.cipher.suites": "AES/CTR/NoPadding",
+ "dfs.namenode.accesstime.precision": "0",
+ "dfs.namenode.write.stale.datanode.ratio": "1.0f",
+ "dfs.datanode.https.address": "0.0.0.0:50475",
+ "dfs.namenode.checkpoint.dir": "/grid/0/hadoop/hdfs/namesecondary",
+ "dfs.namenode.secondary.http-address": "c6401.ambari.apache.org:50090",
+ "nfs.exports.allowed.hosts": "* rw",
+ "dfs.namenode.checkpoint.txns": "1000000",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
+ "dfs.datanode.du.reserved": "33011188224",
+ "dfs.client.read.shortcircuit.streams.cache.size": "4096",
+ "dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",
+ "dfs.web.authentication.kerberos.principal": "HTTP/_HOST@EXAMPLE.COM",
+ "dfs.http.policy": "HTTP_ONLY",
+ "dfs.block.access.token.enable": "true",
+ "dfs.client.retry.policy.enabled": "false",
+ "dfs.secondary.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+ "dfs.datanode.keytab.file": "/etc/security/keytabs/dn.service.keytab",
+ "dfs.namenode.name.dir.restore": "true",
+ "dfs.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",
+ "dfs.journalnode.https-address": "0.0.0.0:8481",
+ "dfs.journalnode.http-address": "0.0.0.0:8480",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "dfs.namenode.avoid.read.stale.datanode": "true",
+ "dfs.hosts.exclude": "/etc/hadoop/conf/dfs.exclude",
+ "dfs.datanode.data.dir.perm": "750",
+ "dfs.encryption.key.provider.uri": "kms://http@c6401.ambari.apache.org:9292/kms",
+ "dfs.replication.max": "50",
+ "dfs.namenode.name.dir": "/grid/0/hadoop/hdfs/namenode"
+ },
+ "ranger-tagsync-site": {
+ "ranger.tagsync.dest.ranger.ssl.config.filename": "/usr/hdp/current/ranger-tagsync/conf/mytruststore.jks",
+ "ranger.tagsync.source.atlasrest.username": "",
+ "ranger.tagsync.logdir": "/var/log/ranger/tagsync",
+ "ranger.tagsync.source.atlasrest.download.interval.millis": "",
+ "ranger.tagsync.keystore.filename": "/usr/hdp/current/ranger-tagsync/conf/rangertagsync.jceks",
+ "ranger.tagsync.source.file.check.interval.millis": "",
+ "ranger.tagsync.source.atlasrest.endpoint": "",
+ "ranger.tagsync.dest.ranger.username": "rangertagsync",
+ "ranger.tagsync.dest.ranger.endpoint": "{{ranger_external_url}}",
+ "ranger.tagsync.kerberos.principal": "rangertagsync/_HOST@EXAMPLE.COM",
+ "ranger.tagsync.kerberos.keytab": "/etc/security/keytabs/rangertagsync.service.keytab",
+ "ranger.tagsync.source.atlas": "false",
+ "ranger.tagsync.source.atlasrest": "false",
+ "ranger.tagsync.source.file": "false",
+ "ranger.tagsync.source.file.filename": ""
+ },
+ "zoo.cfg": {
+ "clientPort": "2181",
+ "autopurge.purgeInterval": "24",
+ "syncLimit": "5",
+ "dataDir": "/grid/0/hadoop/zookeeper",
+ "initLimit": "10",
+ "tickTime": "2000",
+ "autopurge.snapRetainCount": "30"
+ },
+ "hadoop-policy": {
+ "security.job.client.protocol.acl": "*",
+ "security.job.task.protocol.acl": "*",
+ "security.datanode.protocol.acl": "*",
+ "security.namenode.protocol.acl": "*",
+ "security.client.datanode.protocol.acl": "*",
+ "security.inter.tracker.protocol.acl": "*",
+ "security.refresh.usertogroups.mappings.protocol.acl": "hadoop",
+ "security.client.protocol.acl": "*",
+ "security.refresh.policy.protocol.acl": "hadoop",
+ "security.admin.operations.protocol.acl": "hadoop",
+ "security.inter.datanode.protocol.acl": "*"
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.lo
gger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender
.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.security.logg
er}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=
false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# NameNode metrics logging.\n# The default is to retain two namenode-metrics.log files up to 64MB each.\n#\nnamenode.metrics.logger=INFO,NullAppender\nlog4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}\nlog4j.additivity.NameNodeMetricsLog=false\nlog4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log\nlog4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n\nlog4j.appender.NNMETRICSRFA.MaxBackupIndex=1\nlog4j.appender.NNMETRICSRFA.MaxFileSize=64MB\n\n#\n# mapred audit logging\n#\nmapred
.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging leve
ls\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateCh
ange=WARN"
+ },
+ "krb5-conf": {
+ "domains": "",
+ "manage_krb5_conf": "true",
+ "content": "\n[libdefaults]\n renew_lifetime = 7d\n forwardable = true\n default_realm = {{realm}}\n ticket_lifetime = 24h\n dns_lookup_realm = false\n dns_lookup_kdc = false\n default_ccache_name = /tmp/krb5cc_%{uid}\n #default_tgs_enctypes = {{encryption_types}}\n #default_tkt_enctypes = {{encryption_types}}\n{% if domains %}\n[domain_realm]\n{%- for domain in domains.split(',') %}\n {{domain|trim()}} = {{realm}}\n{%- endfor %}\n{% endif %}\n[logging]\n default = FILE:/var/log/krb5kdc.log\n admin_server = FILE:/var/log/kadmind.log\n kdc = FILE:/var/log/krb5kdc.log\n\n[realms]\n {{realm}} = {\n{%- if kdc_hosts > 0 -%}\n{%- set kdc_host_list = kdc_hosts.split(',') -%}\n{%- if kdc_host_list and kdc_host_list|length > 0 %}\n admin_server = {{admin_server_host|default(kdc_host_list[0]|trim(), True)}}\n{%- if kdc_host_list -%}\n{% for kdc_host in kdc_host_list %}\n kdc = {{kdc_host|trim()}}\n{%- endfor -%}\n{% endif %}\n{%- endif %}\n{%- endif %}\n }\n\
n{# Append additional realm declarations below #}",
+ "conf_dir": "/etc"
+ },
+ "core-site": {
+ "net.topology.script.file.name": "/etc/hadoop/conf/topology_script.py",
+ "hadoop.proxyuser.hdfs.groups": "*",
+ "fs.trash.interval": "360",
+ "ipc.server.tcpnodelay": "true",
+ "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec",
+ "ipc.client.idlethreshold": "8000",
+ "io.file.buffer.size": "131072",
+ "hadoop.proxyuser.ambari-server-test_cluster01.groups": "*",
+ "io.serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
+ "hadoop.security.authentication": "kerberos",
+ "mapreduce.jobtracker.webinterface.trusted": "false",
+ "hadoop.proxyuser.hdfs.hosts": "*",
+ "hadoop.proxyuser.HTTP.groups": "users",
+ "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020",
+ "ha.failover-controller.active-standby-elector.zk.op.retries": "120",
+ "hadoop.security.key.provider.path": "kms://http@c6401.ambari.apache.org:9292/kms",
+ "hadoop.security.authorization": "true",
+ "hadoop.http.authentication.simple.anonymous.allowed": "true",
+ "ipc.client.connect.max.retries": "50",
+ "hadoop.security.auth_to_local": "RULE:[1:$1@$0](ambari-qa-test_cluster01@EXAMPLE.COM)s/.*/ambari-qa/\nRULE:[1:$1@$0](hdfs-test_cluster01@EXAMPLE.COM)s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0](dn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](nn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](rangeradmin@EXAMPLE.COM)s/.*/ranger/\nRULE:[2:$1@$0](rangertagsync@EXAMPLE.COM)s/.*/rangertagsync/\nRULE:[2:$1@$0](rangerusersync@EXAMPLE.COM)s/.*/rangerusersync/\nDEFAULT",
+ "hadoop.proxyuser.ambari-server-test_cluster01.hosts": "c6401.ambari.apache.org",
+ "ipc.client.connection.maxidletime": "30000"
+ },
+ "hadoop-env": {
+ "keyserver_port": "",
+ "proxyuser_group": "users",
+ "hdfs_user_nproc_limit": "65536",
+ "hdfs_log_dir_prefix": "/var/log/hadoop",
+ "hdfs_user_nofile_limit": "128000",
+ "hdfs_user": "hdfs",
+ "hdfs_principal_name": "hdfs-test_cluster01@EXAMPLE.COM",
+ "keyserver_host": " ",
+ "namenode_opt_maxnewsize": "128m",
+ "hdfs_user_keytab": "/etc/security/keytabs/hdfs.headless.keytab",
+ "namenode_opt_maxpermsize": "256m",
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appende
d to HADOOP_OPTS when specified\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\n\n{% if java_version < 8 %}\nSHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSiz
e={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\nexport HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms
{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n{% else %}\nSHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{name
node_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\nexport HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARY
NAMENODE_OPTS}\"\n\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n{% endif %}\n\nHADOOP_NFS3_OPTS=\"-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_D
N_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The schedul
ing priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Add database libraries\nJAVA_JDBC_LIBS=\"\"\nif [ -d \"/usr/share/java\" ]; then\n for jarFile in `ls /usr/share/java | grep -E \"(mysql|ojdbc|postgresql|sqljdbc)\" 2>/dev/null`\n do\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\n done\nfi\n\n# Add libraries to the hadoop classpath - some may not need a colon as they already include it\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\n\n{% if is_datanode_max_locked_memory_set %}\n# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. \n# Makes sense to fix only when runing DN as root \nif [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_S
ECURE_DN_USER\" ]; then\n ulimit -l {{datanode_max_locked_memory}}\nfi\n{% endif %}",
+ "namenode_heapsize": "1024m",
+ "namenode_opt_newsize": "128m",
+ "nfsgateway_heapsize": "1024",
+ "dtnode_heapsize": "1024m",
+ "hadoop_root_logger": "INFO,RFA",
+ "hadoop_heapsize": "1024",
+ "hadoop_pid_dir_prefix": "/var/run/hadoop",
+ "namenode_opt_permsize": "128m",
+ "hdfs_tmp_dir": "/tmp"
+ },
+ "zookeeper-log4j": {
+ "content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4
j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN
GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
+ },
+ "ssl-server": {
+ "ssl.server.keystore.location": "/etc/security/serverKeys/keystore.jks",
+ "ssl.server.keystore.keypassword": "bigdata",
+ "ssl.server.truststore.location": "/etc/security/serverKeys/all.jks",
+ "ssl.server.keystore.password": "bigdata",
+ "ssl.server.truststore.password": "bigdata",
+ "ssl.server.truststore.type": "jks",
+ "ssl.server.keystore.type": "jks",
+ "ssl.server.truststore.reload.interval": "10000"
+ },
+ "ranger-site": {},
+ "admin-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nlog4j.rootLogger = warn,xa_log_appender\n\n\n# xa_logger\nlog4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.xa_log_app
ender.file=${logdir}/xa_portal.log\nlog4j.appender.xa_log_appender.datePattern='.'yyyy-MM-dd\nlog4j.appender.xa_log_appender.append=true\nlog4j.appender.xa_log_appender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.xa_log_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n\n# xa_log_appender : category and additivity\nlog4j.category.org.springframework=warn,xa_log_appender\nlog4j.additivity.org.springframework=false\n\nlog4j.category.org.apache.ranger=info,xa_log_appender\nlog4j.additivity.org.apache.ranger=false\n\nlog4j.category.xa=info,xa_log_appender\nlog4j.additivity.xa=false\n\n# perf_logger\nlog4j.appender.perf_appender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.perf_appender.file=${logdir}/ranger_admin_perf.log\nlog4j.appender.perf_appender.datePattern='.'yyyy-MM-dd\nlog4j.appender.perf_appender.append=true\nlog4j.appender.perf_appender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.perf_appender.layout.ConversionPattern=%d [%t]
%m%n\n\n\n# sql_appender\nlog4j.appender.sql_appender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.sql_appender.file=${logdir}/xa_portal_sql.log\nlog4j.appender.sql_appender.datePattern='.'yyyy-MM-dd\nlog4j.appender.sql_appender.append=true\nlog4j.appender.sql_appender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.sql_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n\n\n# sql_appender : category and additivity\nlog4j.category.org.hibernate.SQL=warn,sql_appender\nlog4j.additivity.org.hibernate.SQL=false\n\nlog4j.category.jdbc.sqlonly=fatal,sql_appender\nlog4j.additivity.jdbc.sqlonly=false\n\nlog4j.category.jdbc.sqltiming=warn,sql_appender\nlog4j.additivity.jdbc.sqltiming=false\n\nlog4j.category.jdbc.audit=fatal,sql_appender\nlog4j.additivity.jdbc.audit=false\n\nlog4j.category.jdbc.resultset=fatal,sql_appender\nlog4j.additivity.jdbc.resultset=false\n\nlog4j.category.jdbc.connection=fatal,sql_appender\nlog4j.additivity.jdbc.connection=false"
+ },
+ "tagsync-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nlog4j.rootLogger = info,logFile\n\n# logFile\nlog4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.logFile.file=${logdir}/tagsync
.log\nlog4j.appender.logFile.datePattern='.'yyyy-MM-dd\nlog4j.appender.logFile.layout=org.apache.log4j.PatternLayout\nlog4j.appender.logFile.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n\n\n# console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.Target=System.out\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"
+ },
+ "ranger-hdfs-security": {
+ "ranger.plugin.hdfs.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "ranger.plugin.hdfs.service.name": "{{repo_name}}",
+ "ranger.plugin.hdfs.policy.cache.dir": "/etc/ranger/{{repo_name}}/policycache",
+ "ranger.plugin.hdfs.policy.pollIntervalMs": "30000",
+ "ranger.plugin.hdfs.policy.rest.url": "{{policymgr_mgr_url}}",
+ "ranger.plugin.hdfs.policy.rest.ssl.config.file": "/etc/hadoop/conf/ranger-policymgr-ssl.xml",
+ "xasecure.add-hadoop-authorization": "true"
+ },
+ "usersync-properties": {},
+ "zookeeper-env": {
+ "zk_server_heapsize": "1024m",
+ "zookeeper_keytab_path": "/etc/security/keytabs/zk.service.keytab",
+ "zk_user": "zookeeper",
+ "zk_log_dir": "/var/log/zookeeper",
+ "content": "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}",
+ "zk_pid_dir": "/var/run/zookeeper",
+ "zookeeper_principal_name": "zookeeper/_HOST@EXAMPLE.COM"
+ },
+ "infra-solr-env": {
+ "infra_solr_datadir": "/opt/logsearch_solr/data",
+ "infra_solr_keystore_location": "/etc/security/serverKeys/logsearch.keyStore.jks",
+ "infra_solr_kerberos_name_rules": "DEFAULT",
+ "infra_solr_user": "infra-solr",
+ "infra_solr_maxmem": "1024",
+ "content": "#!/bin/bash\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# By default the script will use JAVA_HOME to determine which java\n# to use, but you can set a specific path for Solr to use without\n# affecting other Java applica
tions on your server/workstation.\nSOLR_JAVA_HOME={{java64_home}}\n\n# Increase Java Min/Max Heap as needed to support your indexing / query needs\nSOLR_JAVA_MEM=\"-Xms{{logsearch_solr_min_mem}}m -Xmx{{logsearch_solr_max_mem}}m\"\n\n# Enable verbose GC logging\nGC_LOG_OPTS=\"-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \\\n-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime\"\n\n# These GC settings have shown to work well for a number of common Solr workloads\nGC_TUNE=\"-XX:NewRatio=3 \\\n-XX:SurvivorRatio=4 \\\n-XX:TargetSurvivorRatio=90 \\\n-XX:MaxTenuringThreshold=8 \\\n-XX:+UseConcMarkSweepGC \\\n-XX:+UseParNewGC \\\n-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \\\n-XX:+CMSScavengeBeforeRemark \\\n-XX:PretenureSizeThreshold=64m \\\n-XX:+UseCMSInitiatingOccupancyOnly \\\n-XX:CMSInitiatingOccupancyFraction=50 \\\n-XX:CMSMaxAbortablePrecleanTime=6000 \\\n-XX:+CMSParallelRemarkEnabled \\\n-XX:+ParallelRefProcEnabled\"\
n\n# Set the ZooKeeper connection string if using an external ZooKeeper ensemble\n# e.g. host1:2181,host2:2181/chroot\n# Leave empty if not using SolrCloud\nZK_HOST=\"{{zookeeper_quorum}}{{logsearch_solr_znode}}\"\n\n# Set the ZooKeeper client timeout (for SolrCloud mode)\nZK_CLIENT_TIMEOUT=\"60000\"\n\n# By default the start script uses \"localhost\"; override the hostname here\n# for production SolrCloud environments to control the hostname exposed to cluster state\n#SOLR_HOST=\"192.168.1.1\"\n\n# By default the start script uses UTC; override the timezone if needed\n#SOLR_TIMEZONE=\"UTC\"\n\n# Set to true to activate the JMX RMI connector to allow remote JMX client applications\n# to monitor the JVM hosting Solr; set to \"false\" to disable that behavior\n# (false is recommended in production environments)\nENABLE_REMOTE_JMX_OPTS=\"true\"\n\n# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here\nRMI_PORT={{logsearch_solr_jmx_port}}\n\n# Anything you add to
the SOLR_OPTS variable will be included in the java\n# start command line as-is, in ADDITION to other options. If you specify the\n# -a option on start script, those options will be appended as well. Examples:\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.clustering.enabled=true\"\n\n# Location where the bin/solr script will save PID files for running instances\n# If not set, the script will create PID files in $SOLR_TIP/bin\nSOLR_PID_DIR={{logsearch_solr_piddir}}\n\n# Path to a directory where Solr creates index files, the specified directory\n# must contain a solr.xml; by default, Solr will use server/solr\nSOLR_HOME={{logsearch_solr_datadir}}\n\n# Solr provides a default Log4J configuration properties file in server/resources\n# however, you may want to customize the log settings and file appender location\n# so you can point the script to use a different log4j.properties f
ile\nLOG4J_PROPS={{logsearch_solr_conf}}/log4j.properties\n\n# Location where Solr should write logs to; should agree with the file appender\n# settings in server/resources/log4j.properties\nSOLR_LOGS_DIR={{logsearch_solr_log_dir}}\n\n# Sets the port Solr binds to, default is 8983\nSOLR_PORT={{logsearch_solr_port}}\n\n# Be sure to update the paths to the correct keystore for your environment\n{% if logsearch_solr_ssl_enabled %}\nSOLR_SSL_KEY_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_KEY_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_TRUST_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_TRUST_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_NEED_CLIENT_AUTH=false\nSOLR_SSL_WANT_CLIENT_AUTH=false\n{% endif %}\n\n# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set\n# and you are using SSL, then the start script will use SOLR_PORT for the SSL port\n#SOLR_SSL_PORT=\n\n{% if security_enabled -%}\nSOLR_HOST=`hostname -f`\nSOLR
_JAAS_FILE={{logsearch_solr_jaas_file}}\nSOLR_KERB_KEYTAB={{logsearch_solr_web_kerberos_keytab}}\nSOLR_KERB_PRINCIPAL={{logsearch_solr_web_kerberos_principal}}\nSOLR_KERB_NAME_RULES={{logsearch_solr_kerberos_name_rules}}\n\nSOLR_AUTHENTICATION_CLIENT_CONFIGURER=\"org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer\"\nSOLR_AUTHENTICATION_OPTS=\" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=$SOLR_JAAS_FILE -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST} -Dsolr.kerberos.name.rules=${SOLR_KERB_NAME_RULES}\"\n{% endif %}",
+ "infra_solr_pid_dir": "/var/run/ambari-infra-solr",
+ "infra_solr_truststore_password": "bigdata",
+ "infra_solr_truststore_type": "jks",
+ "infra_solr_keystore_type": "jks",
+ "infra_solr_log_dir": "/var/log/ambari-infra-solr",
+ "infra_solr_web_kerberos_keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "infra_solr_ssl_enabled": "false",
+ "infra_solr_client_log_dir": "/var/log/ambari-infra-solr-client",
+ "infra_solr_web_kerberos_principal": "HTTP/_HOST@EXAMPLE.COM",
+ "infra_solr_znode": "/ambari-solr",
+ "infra_solr_keystore_password": "bigdata",
+ "infra_solr_port": "8886",
+ "infra_solr_kerberos_principal": "infra-solr/_HOST@EXAMPLE.COM",
+ "infra_solr_jmx_port": "18886",
+ "infra_solr_truststore_location": "/etc/security/serverKeys/infra-solr.trustStore.jks",
+ "infra_solr_minmem": "512",
+ "infra_solr_kerberos_keytab": "/etc/security/keytabs/infra-solr.service.keytab"
+ },
+ "infra-solr-client-log4j": {
+ "infra_solr_client_log_dir" : "/var/log/ambari-infra-solr-client",
+ "content": "# Copyright 2011 The Apache Software Foundation\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlog4j.rootLogger=INFO,file,stdout,stderr\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender
.file.File={{logsearch_solr_client_log}}\nlog4j.appender.file.MaxFileSize=80MB\nlog4j.appender.file.MaxBackupIndex=60\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.Threshold=INFO\nlog4j.appender.stdout.Target=System.out\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=%m%n\n\nlog4j.appender.stderr=org.apache.log4j.ConsoleAppender\nlog4j.appender.stderr.Threshold=ERROR\nlog4j.appender.stderr.Target=System.err\nlog4j.appender.stderr.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stderr.layout.ConversionPattern=%m%n"
+ },
+ "cluster-env": {
+ "security_enabled": "true",
+ "override_uid": "true",
+ "fetch_nonlocal_groups": "true",
+ "one_dir_per_partition": "true",
+ "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}",
+ "ignore_groupsusers_create": "false",
+ "alerts_repeat_tolerance": "1",
+ "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
+ "kerberos_domain": "EXAMPLE.COM",
+ "manage_dirs_on_root": "true",
+ "recovery_lifetime_max_count": "1024",
+ "recovery_type": "AUTO_START",
+ "ignore_bad_mounts": "false",
+ "recovery_window_in_minutes": "60",
+ "user_group": "hadoop",
+ "stack_tools": "{\n \"stack_selector\": [\"hdp-select\", \"/usr/bin/hdp-select\", \"hdp-select\"],\n \"conf_selector\": [\"conf-select\", \"/usr/bin/conf-select\", \"conf-select\"]\n}",
+ "recovery_retry_interval": "5",
+ "stack_features": "{\n \"stack_features\": [\n {\n \"name\": \"snappy\",\n \"description\": \"Snappy compressor/decompressor support\",\n \"min_version\": \"2.0.0.0\",\n \"max_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"lzo\",\n \"description\": \"LZO libraries support\",\n \"min_version\": \"2.2.1.0\"\n },\n {\n \"name\": \"express_upgrade\",\n \"description\": \"Express upgrade support\",\n \"min_version\": \"2.1.0.0\"\n },\n {\n \"name\": \"rolling_upgrade\",\n \"description\": \"Rolling upgrade support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"config_versioning\",\n \"description\": \"Configurable versions support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"datanode_non_root\",\n \"description\": \"DataNode running as non-root support (AMBARI-7615)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\
": \"remove_ranger_hdfs_plugin_env\",\n \"description\": \"HDFS removes Ranger env files (AMBARI-14299)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger\",\n \"description\": \"Ranger Service support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_tagsync_component\",\n \"description\": \"Ranger Tagsync component support (AMBARI-14383)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"phoenix\",\n \"description\": \"Phoenix Service support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"nfs\",\n \"description\": \"NFS support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"tez_for_spark\",\n \"description\": \"Tez dependency for Spark\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"timeline_state_store\",\n \"description\": \"Yarn application timeline-se
rvice supports state store property (AMBARI-11442)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"copy_tarball_to_hdfs\",\n \"description\": \"Copy tarball to HDFS support (AMBARI-12113)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"spark_16plus\",\n \"description\": \"Spark 1.6+\",\n \"min_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"spark_thriftserver\",\n \"description\": \"Spark Thrift Server\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"storm_kerberos\",\n \"description\": \"Storm Kerberos support (AMBARI-7570)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"storm_ams\",\n \"description\": \"Storm AMS integration (AMBARI-10710)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"create_kafka_broker_id\",\n \"description\": \"Ambari should create Kafka Broker Id (AMBARI-12678)\",\n \"min_version\": \"2.2
.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_listeners\",\n \"description\": \"Kafka listeners (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_kerberos\",\n \"description\": \"Kafka Kerberos support (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"pig_on_tez\",\n \"description\": \"Pig on Tez support (AMBARI-7863)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_usersync_non_root\",\n \"description\": \"Ranger Usersync as non-root user (AMBARI-10416)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger_audit_db_support\",\n \"description\": \"Ranger Audit to DB support\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"accumulo_kerberos_user_auth\",\n \"description\": \"Accumulo Kerberos User Auth (AMBARI-10163)\",\n
\"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"knox_versioned_data_dir\",\n \"description\": \"Use versioned data dir for Knox (AMBARI-13164)\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"knox_sso_topology\",\n \"description\": \"Knox SSO Topology support (AMBARI-13975)\",\n \"min_version\": \"2.3.8.0\"\n },\n {\n \"name\": \"atlas_rolling_upgrade\",\n \"description\": \"Rolling upgrade support for Atlas\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"oozie_admin_user\",\n \"description\": \"Oozie install user as an Oozie admin user (AMBARI-7976)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_create_hive_tez_configs\",\n \"description\": \"Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_setup_shared_lib\",\n \"description\": \"Oozie
setup tools used to shared Oozie lib to HDFS (AMBARI-7240)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_host_kerberos\",\n \"description\": \"Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)\",\n \"min_version\": \"2.0.0.0\",\n \"max_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"falcon_extensions\",\n \"description\": \"Falcon Extension\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_upgrade_schema\",\n \"description\": \"Hive metastore upgrade schema support (AMBARI-11176)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server_interactive\",\n \"description\": \"Hive server interactive support (AMBARI-15573)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_webhcat_specific_configs\",\n \"description\": \"Hive webhcat specific configurations support (AMBARI-12364)\",\n \"min_ver
sion\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_purge_table\",\n \"description\": \"Hive purge table support (AMBARI-12260)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server2_kerberized_env\",\n \"description\": \"Hive server2 working on kerberized environment (AMBARI-13749)\",\n \"min_version\": \"2.2.3.0\",\n \"max_version\": \"2.2.5.0\"\n },\n {\n \"name\": \"hive_env_heapsize\",\n \"description\": \"Hive heapsize property defined in hive-env (AMBARI-12801)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_kms_hsm_support\",\n \"description\": \"Ranger KMS HSM support (AMBARI-15752)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_log4j_support\",\n \"description\": \"Ranger supporting log-4j properties (AMBARI-15681)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kerberos_support\",\n \"desc
ription\": \"Ranger Kerberos support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_site_support\",\n \"description\": \"Hive Metastore site support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_usersync_password_jceks\",\n \"description\": \"Saving Ranger Usersync credentials in jceks\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_install_infra_client\",\n \"description\": \"LogSearch Service support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hbase_home_directory\",\n \"description\": \"Hbase home directory in HDFS needed for HBASE backup\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_livy\",\n \"description\": \"Livy as slave component of spark\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"atlas_ranger_plugin_support\",\n \"description\": \"Atlas Ranger plug
in support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_pid_support\",\n \"description\": \"Ranger Service support pid generation AMBARI-16756\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kms_pid_support\",\n \"description\": \"Ranger KMS Service support pid generation\",\n \"min_version\": \"2.5.0.0\"\n }\n ]\n}",
+ "recovery_enabled": "true",
+ "smokeuser_principal_name": "ambari-qa-test_cluster01@EXAMPLE.COM",
+ "recovery_max_count": "6",
+ "stack_root": "/usr/hdp",
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "ambari_principal_name": "ambari-server-test_cluster01@EXAMPLE.COM",
+ "managed_hdfs_resource_property_names": "",
+ "smokeuser": "ambari-qa"
+ }
+ }
+}
\ No newline at end of file
[36/50] [abbrv] ambari git commit: AMBARI-19091. HDP 3.0 TP -
bootstrap the stack with metainfo, cluster-env, repos, and hooks (alejandro)
Posted by nc...@apache.org.
AMBARI-19091. HDP 3.0 TP - bootstrap the stack with metainfo, cluster-env, repos, and hooks (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/757267fc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/757267fc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/757267fc
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 757267fc6c62dff71aeb162976de39278d40aa04
Parents: 9a938e6
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Fri Dec 2 13:26:49 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Tue Dec 6 16:51:28 2016 -0800
----------------------------------------------------------------------
.../HDP/3.0/configuration/cluster-env.xml | 293 +++++++++++++++++
.../HDP/3.0/hooks/after-INSTALL/scripts/hook.py | 37 +++
.../3.0/hooks/after-INSTALL/scripts/params.py | 97 ++++++
.../scripts/shared_initialization.py | 111 +++++++
.../hooks/before-ANY/files/changeToSecureUid.sh | 53 +++
.../HDP/3.0/hooks/before-ANY/scripts/hook.py | 36 ++
.../HDP/3.0/hooks/before-ANY/scripts/params.py | 231 +++++++++++++
.../before-ANY/scripts/shared_initialization.py | 226 +++++++++++++
.../3.0/hooks/before-INSTALL/scripts/hook.py | 37 +++
.../3.0/hooks/before-INSTALL/scripts/params.py | 113 +++++++
.../scripts/repo_initialization.py | 68 ++++
.../scripts/shared_initialization.py | 37 +++
.../3.0/hooks/before-RESTART/scripts/hook.py | 29 ++
.../hooks/before-START/files/checkForFormat.sh | 65 ++++
.../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 19285850 bytes
.../before-START/files/task-log4j.properties | 134 ++++++++
.../hooks/before-START/files/topology_script.py | 66 ++++
.../HDP/3.0/hooks/before-START/scripts/hook.py | 39 +++
.../3.0/hooks/before-START/scripts/params.py | 326 +++++++++++++++++++
.../before-START/scripts/rack_awareness.py | 47 +++
.../scripts/shared_initialization.py | 191 +++++++++++
.../templates/commons-logging.properties.j2 | 43 +++
.../templates/exclude_hosts_list.j2 | 21 ++
.../templates/hadoop-metrics2.properties.j2 | 105 ++++++
.../before-START/templates/health_check.j2 | 81 +++++
.../templates/include_hosts_list.j2 | 21 ++
.../templates/topology_mappings.data.j2 | 24 ++
.../main/resources/stacks/HDP/3.0/kerberos.json | 78 +++++
.../main/resources/stacks/HDP/3.0/metainfo.xml | 24 ++
.../HDP/3.0/properties/stack_features.json | 323 ++++++++++++++++++
.../stacks/HDP/3.0/properties/stack_tools.json | 4 +
.../resources/stacks/HDP/3.0/repos/repoinfo.xml | 132 ++++++++
.../main/resources/stacks/HDP/3.0/widgets.json | 95 ++++++
33 files changed, 3187 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/configuration/cluster-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/configuration/cluster-env.xml
new file mode 100644
index 0000000..93680bf
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/configuration/cluster-env.xml
@@ -0,0 +1,293 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>recovery_enabled</name>
+ <value>true</value>
+ <description>Auto start enabled or not for this cluster.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_type</name>
+ <value>AUTO_START</value>
+ <description>Auto start type.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_lifetime_max_count</name>
+ <value>1024</value>
+ <description>Auto start lifetime maximum count of recovery attempt allowed per host component. This is reset when agent is restarted.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_max_count</name>
+ <value>6</value>
+ <description>Auto start maximum count of recovery attempt allowed per host component in a window. This is reset when agent is restarted.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_window_in_minutes</name>
+ <value>60</value>
+ <description>Auto start recovery window size in minutes.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>recovery_retry_interval</name>
+ <value>5</value>
+ <description>Auto start recovery retry gap between tries per host component.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>security_enabled</name>
+ <value>false</value>
+ <description>Hadoop Security</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kerberos_domain</name>
+ <value>EXAMPLE.COM</value>
+ <description>Kerberos realm.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>ignore_groupsusers_create</name>
+ <display-name>Skip group modifications during install</display-name>
+ <value>false</value>
+ <property-type>ADDITIONAL_USER_PROPERTY</property-type>
+ <description>Whether to ignore failures on users and group creation</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>sysprep_skip_create_users_and_groups</name>
+ <display-name>Whether to skip creating users and groups in a sysprepped cluster</display-name>
+ <value>false</value>
+ <property-type>ADDITIONAL_USER_PROPERTY</property-type>
+ <description>Whether to skip creating users and groups in a sysprepped cluster</description>
+ <value-attributes>
+ <overridable>true</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>sysprep_skip_copy_fast_jar_hdfs</name>
+ <display-name>Whether to skip copying the tarballs to HDFS on a sysprepped cluster</display-name>
+ <value>false</value>
+ <description>Whether to skip copying the tarballs to HDFS on a sysprepped cluster, during both fresh install and stack upgrade</description>
+ <value-attributes>
+ <overridable>true</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>sysprep_skip_copy_tarballs_hdfs</name>
+ <display-name>Whether to skip copying the tarballs to HDFS on a sysprepped cluster</display-name>
+ <value>false</value>
+ <description>Whether to skip copying the tarballs to HDFS on a sysprepped cluster, during both fresh install and stack upgrade</description>
+ <value-attributes>
+ <overridable>true</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>sysprep_skip_copy_oozie_share_lib_to_hdfs</name>
+ <display-name>Whether to skip copying the Oozie share lib to HDFS on sysprepped cluster</display-name>
+ <value>false</value>
+ <description>Whether to skip copying the Oozie share lib to HDFS on sysprepped cluster, during both fresh install and stack upgrade</description>
+ <value-attributes>
+ <overridable>true</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>smokeuser</name>
+ <display-name>Smoke User</display-name>
+ <value>ambari-qa</value>
+ <property-type>USER</property-type>
+ <description>User executing service checks</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>smokeuser_keytab</name>
+ <value>/etc/security/keytabs/smokeuser.headless.keytab</value>
+ <description>Path to smoke test user keytab file</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>user_group</name>
+ <display-name>Hadoop Group</display-name>
+ <value>hadoop</value>
+ <property-type>GROUP</property-type>
+ <description>Hadoop user group.</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>repo_suse_rhel_template</name>
+ <value>[{{repo_id}}]
+name={{repo_id}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+
+path=/
+enabled=1
+gpgcheck=0</value>
+ <description>Template of repositories for rhel and suse.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>repo_ubuntu_template</name>
+ <value>{{package_type}} {{base_url}} {{components}}</value>
+ <description>Template of repositories for ubuntu.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>override_uid</name>
+ <value>true</value>
+ <property-type>ADDITIONAL_USER_PROPERTY</property-type>
+ <display-name>Have Ambari manage UIDs</display-name>
+ <description>Have Ambari manage UIDs</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>fetch_nonlocal_groups</name>
+ <value>true</value>
+ <display-name>Ambari fetch nonlocal groups</display-name>
+ <description>Ambari requires fetching all the groups. This can be slow
+ on envs with enabled ldap. Setting this option to false will enable Ambari,
+ to skip user/group management connected with ldap groups.</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>managed_hdfs_resource_property_names</name>
+ <value/>
+ <description>Comma separated list of property names with HDFS resource paths.
+ Resource from this list will be managed even if it is marked as not managed in the stack</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <!-- Define stack_tools property in the base stack. DO NOT override this property for each stack version -->
+ <property>
+ <name>stack_tools</name>
+ <value/>
+ <description>Stack specific tools</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>stack_tools.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ <read-only>true</read-only>
+ <overridable>false</overridable>
+ <visible>false</visible>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <!-- Define stack_features property in the base stack. DO NOT override this property for each stack version -->
+ <property>
+ <name>stack_features</name>
+ <value/>
+ <description>List of features supported by the stack</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>stack_features.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ <read-only>true</read-only>
+ <overridable>false</overridable>
+ <visible>false</visible>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>stack_root</name>
+ <value>/usr/hdp</value>
+ <description>Stack root folder</description>
+ <value-attributes>
+ <read-only>true</read-only>
+ <overridable>false</overridable>
+ <visible>false</visible>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>alerts_repeat_tolerance</name>
+ <value>1</value>
+ <description>The number of consecutive alerts required to transition an alert from the SOFT to the HARD state.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>ignore_bad_mounts</name>
+ <value>false</value>
+ <description>For properties handled by handle_mounted_dirs this will make Ambari not to create any directories.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>manage_dirs_on_root</name>
+ <value>true</value>
+ <description>For properties handled by handle_mounted_dirs this will make Ambari to manage (create and set permissions) unknown directories on / partition</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>one_dir_per_partition</name>
+ <value>false</value>
+ <description>For properties handled by handle_mounted_dirs this will make Ambari </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hide_yarn_memory_widget</name>
+ <value>false</value>
+ <description>YARN Memory widget should be hidden by default on the dashboard.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>agent_mounts_ignore_list</name>
+ <value/>
+ <description>Comma separated list of the mounts which would be ignored by Ambari during property values suggestion by Stack Advisor</description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <visible>true</visible>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/hook.py
new file mode 100644
index 0000000..8a583b3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/hook.py
@@ -0,0 +1,37 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script.hook import Hook
+from shared_initialization import link_configs
+from shared_initialization import setup_config
+from shared_initialization import setup_stack_symlinks
+
+class AfterInstallHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ env.set_params(params)
+ setup_stack_symlinks()
+ setup_config()
+
+ link_configs(self.stroutfile)
+
+if __name__ == "__main__":
+ AfterInstallHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/params.py
new file mode 100644
index 0000000..566f5b3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/params.py
@@ -0,0 +1,97 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions import default
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format_jvm_option
+from resource_management.libraries.functions.version import format_stack_version
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+dfs_type = default("/commandParams/dfs_type", "")
+stack_root = Script.get_stack_root()
+
+is_parallel_execution_enabled = int(default("/agentConfigParams/agent/parallel_execution", 0)) == 1
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+sudo = AMBARI_SUDO_BINARY
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# current host stack version
+current_version = default("/hostLevelParams/current_version", None)
+
+# default hadoop params
+mapreduce_libs_path = format("{stack_root}/current/hadoop-mapreduce-client/*")
+hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
+hadoop_conf_empty_dir = None
+
+versioned_stack_root = format('{stack_root}/current')
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#java params
+java_home = config['hostLevelParams']['java_home']
+
+#hadoop params
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger']
+
+jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+#users and groups
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
+
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+has_namenode = not len(namenode_host) == 0
+
+if has_namenode or dfs_type == 'HCFS':
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+
+link_configs_lock_file = os.path.join(tmp_dir, "link_configs_lock_file")
+stack_select_lock_file = os.path.join(tmp_dir, "stack_select_lock_file")
+
+upgrade_suspended = default("/roleParams/upgrade_suspended", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/shared_initialization.py
new file mode 100644
index 0000000..e9f2283
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -0,0 +1,111 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+
+import ambari_simplejson as json
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.script import Script
+
+
+def setup_stack_symlinks():
+ """
+ Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a
+ stack version, such as "2.3". This should always be called after a component has been
+ installed to ensure that all HDP pointers are correct. The stack upgrade logic does not
+ interact with this since it's done via a custom command and will not trigger this hook.
+ :return:
+ """
+ import params
+ if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
+ # try using the exact version first, falling back in just the stack if it's not defined
+ # which would only be during an intial cluster installation
+ version = params.current_version if params.current_version is not None else params.stack_version_unformatted
+
+ if not params.upgrade_suspended:
+ if params.host_sys_prepped:
+ Logger.warning("Skipping running stack-selector-tool for stack {0} as its a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.".format(version))
+ return
+ # On parallel command execution this should be executed by a single process at a time.
+ with FcntlBasedProcessLock(params.stack_select_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
+ stack_select.select_all(version)
+
+def setup_config():
+ import params
+ stackversion = params.stack_version_unformatted
+ Logger.info("FS Type: {0}".format(params.dfs_type))
+
+ is_hadoop_conf_dir_present = False
+ if hasattr(params, "hadoop_conf_dir") and params.hadoop_conf_dir is not None and os.path.exists(params.hadoop_conf_dir):
+ is_hadoop_conf_dir_present = True
+ else:
+ Logger.warning("Parameter hadoop_conf_dir is missing or directory does not exist. This is expected if this host does not have any Hadoop components.")
+
+ if is_hadoop_conf_dir_present and (params.has_namenode or stackversion.find('Gluster') >= 0 or params.dfs_type == 'HCFS'):
+ # create core-site only if the hadoop config diretory exists
+ XmlConfig("core-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.hdfs_user,
+ group=params.user_group,
+ only_if=format("ls {hadoop_conf_dir}"))
+
+
+def load_version(struct_out_file):
+ """
+ Load version from file. Made a separate method for testing
+ """
+ json_version = None
+ try:
+ if os.path.exists(struct_out_file):
+ with open(struct_out_file, 'r') as fp:
+ json_info = json.load(fp)
+ json_version = json_info['version']
+ except:
+ pass
+
+ return json_version
+
+
+def link_configs(struct_out_file):
+ """
+ Links configs, only on a fresh install of HDP-2.3 and higher
+ """
+ import params
+
+ if not Script.is_stack_greater_or_equal("2.3"):
+ Logger.info("Can only link configs for HDP-2.3 and higher.")
+ return
+
+ json_version = load_version(struct_out_file)
+
+ if not json_version:
+ Logger.info("Could not load 'version' from {0}".format(struct_out_file))
+ return
+
+ # On parallel command execution this should be executed by a single process at a time.
+ with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True):
+ for k, v in conf_select.get_package_dirs().iteritems():
+ conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/files/changeToSecureUid.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/files/changeToSecureUid.sh b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/files/changeToSecureUid.sh
new file mode 100644
index 0000000..08542c4
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/files/changeToSecureUid.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+username=$1
+directories=$2
+
+function find_available_uid() {
+ for ((i=1001; i<=2000; i++))
+ do
+ grep -q $i /etc/passwd
+ if [ "$?" -ne 0 ]
+ then
+ newUid=$i
+ break
+ fi
+ done
+}
+
+find_available_uid
+
+if [ $newUid -eq 0 ]
+then
+ echo "Failed to find Uid between 1000 and 2000"
+ exit 1
+fi
+
+set -e
+
+dir_array=($(echo $directories | sed 's/,/\n/g'))
+old_uid=$(id -u $username)
+sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
+echo "Changing uid of $username from $old_uid to $newUid"
+echo "Changing directory permisions for ${dir_array[@]}"
+$sudo_prefix usermod -u $newUid $username && for dir in ${dir_array[@]} ; do ls $dir 2> /dev/null && echo "Changing permission for $dir" && $sudo_prefix chown -Rh $newUid $dir ; done
+exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/hook.py
new file mode 100644
index 0000000..c34be0b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/hook.py
@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from shared_initialization import *
+
+class BeforeAnyHook(Hook):
+
+ def hook(self, env):
+ import params
+ env.set_params(params)
+
+ setup_users()
+ if params.has_namenode or params.dfs_type == 'HCFS':
+ setup_hadoop_env()
+ setup_java()
+
+if __name__ == "__main__":
+ BeforeAnyHook().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py
new file mode 100644
index 0000000..f70c8e9
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py
@@ -0,0 +1,231 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import collections
+import re
+import os
+
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions import default
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format_jvm_option
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.expect import expect
+from ambari_commons.os_check import OSCheck
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+dfs_type = default("/commandParams/dfs_type", "")
+stack_root = Script.get_stack_root()
+
+artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
+jdk_name = default("/hostLevelParams/jdk_name", None)
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+jdk_location = config['hostLevelParams']['jdk_location']
+
+sudo = AMBARI_SUDO_BINARY
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+upgrade_type = Script.get_upgrade_type(default("/commandParams/upgrade_type", ""))
+version = default("/commandParams/version", None)
+# Handle upgrade and downgrade
+if (upgrade_type is not None) and version:
+ stack_version_formatted = format_stack_version(version)
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+
+# Some datanode settings
+dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None)
+dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None)
+dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None)
+dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None)
+secure_dn_ports_are_in_use = False
+
+def get_port(address):
+ """
+ Extracts port from the address like 0.0.0.0:1019
+ """
+ if address is None:
+ return None
+ m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address)
+ if m is not None:
+ return int(m.group(2))
+ else:
+ return None
+
+def is_secure_port(port):
+ """
+ Returns True if port is root-owned at *nix systems
+ """
+ if port is not None:
+ return port < 1024
+ else:
+ return False
+
+# hadoop default params
+mapreduce_libs_path = format("{stack_root}/current/hadoop-mapreduce-client/*")
+
+# upgrades would cause these directories to have a version instead of "current"
+# which would cause a lot of problems when writing out hadoop-env.sh; instead
+# force the use of "current" in the hook
+hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000")
+hadoop_home = stack_select.get_hadoop_dir("home", force_latest_on_upgrade=True)
+hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True)
+
+hadoop_conf_empty_dir = None
+hadoop_secure_dn_user = hdfs_user
+hadoop_dir = "/etc/hadoop"
+versioned_stack_root = format('{stack_root}/current')
+hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
+datanode_max_locked_memory = config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']
+is_datanode_max_locked_memory_set = not is_empty(config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory'])
+
+if not security_enabled:
+ hadoop_secure_dn_user = '""'
+else:
+ dfs_dn_port = get_port(dfs_dn_addr)
+ dfs_dn_http_port = get_port(dfs_dn_http_addr)
+ dfs_dn_https_port = get_port(dfs_dn_https_addr)
+ # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
+ if dfs_http_policy == "HTTPS_ONLY":
+ secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port)
+ elif dfs_http_policy == "HTTP_AND_HTTPS":
+ secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) or is_secure_port(dfs_dn_https_port)
+ else: # params.dfs_http_policy == "HTTP_ONLY" or not defined:
+ secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port)
+ if secure_dn_ports_are_in_use:
+ hadoop_secure_dn_user = hdfs_user
+ else:
+ hadoop_secure_dn_user = '""'
+
+#hadoop params
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger']
+
+jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+hadoop_env_sh_template = config['configurations']['hadoop-env']['content']
+
+#users and groups
+hbase_user = config['configurations']['hbase-env']['hbase_user']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+gmetad_user = config['configurations']['ganglia-env']["gmetad_user"]
+gmond_user = config['configurations']['ganglia-env']["gmond_user"]
+tez_user = config['configurations']['tez-env']["tez_user"]
+oozie_user = config['configurations']['oozie-env']["oozie_user"]
+falcon_user = config['configurations']['falcon-env']["falcon_user"]
+ranger_user = config['configurations']['ranger-env']["ranger_user"]
+zeppelin_user = config['configurations']['zeppelin-env']["zeppelin_user"]
+zeppelin_group = config['configurations']['zeppelin-env']["zeppelin_group"]
+
+user_group = config['configurations']['cluster-env']['user_group']
+
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+falcon_server_hosts = default("/clusterHostInfo/falcon_server_hosts", [])
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+zeppelin_master_hosts = default("/clusterHostInfo/zeppelin_master_hosts", [])
+
+has_namenode = not len(namenode_host) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_tez = 'tez-site' in config['configurations']
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_falcon_server_hosts = not len(falcon_server_hosts) == 0
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+has_zeppelin_master = not len(zeppelin_master_hosts) == 0
+
+if has_namenode or dfs_type == 'HCFS':
+ hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+
+hbase_tmp_dir = "/tmp/hbase-hbase"
+
+proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users")
+ranger_group = config['configurations']['ranger-env']['ranger_group']
+dfs_cluster_administrators_group = config['configurations']['hdfs-site']["dfs.cluster.administrators"]
+
+sysprep_skip_create_users_and_groups = default("/configurations/cluster-env/sysprep_skip_create_users_and_groups", False)
+ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False)
+fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"]
+
+smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
+if has_hbase_masters:
+ hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}")
+#repo params
+repo_info = config['hostLevelParams']['repo_info']
+service_repo_info = default("/hostLevelParams/service_repo_info",None)
+
+user_to_groups_dict = collections.defaultdict(lambda:[user_group])
+user_to_groups_dict[smoke_user] = [proxyuser_group]
+if has_ganglia_server:
+ user_to_groups_dict[gmond_user] = [gmond_user]
+ user_to_groups_dict[gmetad_user] = [gmetad_user]
+if has_tez:
+ user_to_groups_dict[tez_user] = [proxyuser_group]
+if has_oozie_server:
+ user_to_groups_dict[oozie_user] = [proxyuser_group]
+if has_falcon_server_hosts:
+ user_to_groups_dict[falcon_user] = [proxyuser_group]
+if has_ranger_admin:
+ user_to_groups_dict[ranger_user] = [ranger_group]
+if has_zeppelin_master:
+ user_to_groups_dict[zeppelin_user] = [zeppelin_group, user_group]
+
+user_to_gid_dict = collections.defaultdict(lambda:user_group)
+
+user_list = json.loads(config['hostLevelParams']['user_list'])
+group_list = json.loads(config['hostLevelParams']['group_list'])
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+tez_am_view_acls = config['configurations']['tez-site']["tez.am.view-acls"]
+override_uid = str(default("/configurations/cluster-env/override_uid", "true")).lower()
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py
new file mode 100644
index 0000000..320872e
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/shared_initialization.py
@@ -0,0 +1,226 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import getpass
+import tempfile
+from copy import copy
+from resource_management.libraries.functions.version import compare_versions
+from resource_management import *
+
+def setup_users():
+ """
+ Creates users before cluster installation
+ """
+ import params
+
+ should_create_users_and_groups = False
+ if params.host_sys_prepped:
+ should_create_users_and_groups = not params.sysprep_skip_create_users_and_groups
+ else:
+ should_create_users_and_groups = not params.ignore_groupsusers_create
+
+ if should_create_users_and_groups:
+ for group in params.group_list:
+ Group(group,
+ )
+
+ for user in params.user_list:
+ User(user,
+ gid = params.user_to_gid_dict[user],
+ groups = params.user_to_groups_dict[user],
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ )
+
+ if params.override_uid == "true":
+ set_uid(params.smoke_user, params.smoke_user_dirs)
+ else:
+ Logger.info('Skipping setting uid for smoke user as host is sys prepped')
+ else:
+ Logger.info('Skipping creation of User and Group as host is sys prepped or ignore_groupsusers_create flag is on')
+ pass
+
+
+ if params.has_hbase_masters:
+ Directory (params.hbase_tmp_dir,
+ owner = params.hbase_user,
+ mode=0775,
+ create_parents = True,
+ cd_access="a",
+ )
+ if params.override_uid == "true":
+ set_uid(params.hbase_user, params.hbase_user_dirs)
+ else:
+ Logger.info('Skipping setting uid for hbase user as host is sys prepped')
+
+ if should_create_users_and_groups:
+ if params.has_namenode:
+ create_dfs_cluster_admins()
+ if params.has_tez and params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.3') >= 0:
+ create_tez_am_view_acls()
+ else:
+ Logger.info('Skipping setting dfs cluster admin and tez view acls as host is sys prepped')
+
+def create_dfs_cluster_admins():
+ """
+ dfs.cluster.administrators support format <comma-delimited list of usernames><space><comma-delimited list of group names>
+ """
+ import params
+
+ groups_list = create_users_and_groups(params.dfs_cluster_administrators_group)
+
+ User(params.hdfs_user,
+ groups = params.user_to_groups_dict[params.hdfs_user] + groups_list,
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ )
+
+def create_tez_am_view_acls():
+
+ """
+ tez.am.view-acls support format <comma-delimited list of usernames><space><comma-delimited list of group names>
+ """
+ import params
+
+ if not params.tez_am_view_acls.startswith("*"):
+ create_users_and_groups(params.tez_am_view_acls)
+
+def create_users_and_groups(user_and_groups):
+
+ import params
+
+ parts = re.split('\s', user_and_groups)
+ if len(parts) == 1:
+ parts.append("")
+
+ users_list = parts[0].split(",") if parts[0] else []
+ groups_list = parts[1].split(",") if parts[1] else []
+
+ if users_list:
+ User(users_list,
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ )
+
+ if groups_list:
+ Group(copy(groups_list),
+ )
+ return groups_list
+
+def set_uid(user, user_dirs):
+ """
+ user_dirs - comma separated directories
+ """
+ import params
+
+ File(format("{tmp_dir}/changeUid.sh"),
+ content=StaticFile("changeToSecureUid.sh"),
+ mode=0555)
+ ignore_groupsusers_create_str = str(params.ignore_groupsusers_create).lower()
+ Execute(format("{tmp_dir}/changeUid.sh {user} {user_dirs}"),
+ not_if = format("(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})"))
+
+def setup_hadoop_env():
+ import params
+ stackversion = params.stack_version_unformatted
+ Logger.info("FS Type: {0}".format(params.dfs_type))
+ if params.has_namenode or stackversion.find('Gluster') >= 0 or params.dfs_type == 'HCFS':
+ if params.security_enabled:
+ tc_owner = "root"
+ else:
+ tc_owner = params.hdfs_user
+
+ # create /etc/hadoop
+ Directory(params.hadoop_dir, mode=0755)
+
+ # HDP < 2.2 used a conf -> conf.empty symlink for /etc/hadoop/
+ if Script.is_stack_less_than("2.2"):
+ Directory(params.hadoop_conf_empty_dir, create_parents = True, owner="root",
+ group=params.user_group )
+
+ Link(params.hadoop_conf_dir, to=params.hadoop_conf_empty_dir,
+ not_if=format("ls {hadoop_conf_dir}"))
+
+ # write out hadoop-env.sh, but only if the directory exists
+ if os.path.exists(params.hadoop_conf_dir):
+ File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'), owner=tc_owner,
+ group=params.user_group,
+ content=InlineTemplate(params.hadoop_env_sh_template))
+
+ # Create tmp dir for java.io.tmpdir
+ # Handle a situation when /tmp is set to noexec
+ Directory(params.hadoop_java_io_tmpdir,
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=01777
+ )
+
+def setup_java():
+ """
+ Installs jdk using specific params, that comes from ambari-server
+ """
+ import params
+
+ java_exec = format("{java_home}/bin/java")
+
+ if not os.path.isfile(java_exec):
+ if not params.jdk_name: # if custom jdk is used.
+ raise Fail(format("Unable to access {java_exec}. Confirm you have copied jdk to this host."))
+
+ jdk_curl_target = format("{tmp_dir}/{jdk_name}")
+ java_dir = os.path.dirname(params.java_home)
+
+ Directory(params.artifact_dir,
+ create_parents = True,
+ )
+
+ File(jdk_curl_target,
+ content = DownloadSource(format("{jdk_location}/{jdk_name}")),
+ not_if = format("test -f {jdk_curl_target}")
+ )
+
+ tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir)
+
+ try:
+ if params.jdk_name.endswith(".bin"):
+ chmod_cmd = ("chmod", "+x", jdk_curl_target)
+ install_cmd = format("cd {tmp_java_dir} && echo A | {jdk_curl_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
+ elif params.jdk_name.endswith(".gz"):
+ chmod_cmd = ("chmod","a+x", java_dir)
+ install_cmd = format("cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}")
+
+ Directory(java_dir
+ )
+
+ Execute(chmod_cmd,
+ sudo = True,
+ )
+
+ Execute(install_cmd,
+ )
+
+ finally:
+ Directory(tmp_java_dir, action="delete")
+
+ File(format("{java_home}/bin/java"),
+ mode=0755,
+ cd_access="a",
+ )
+ Execute(('chmod', '-R', '755', params.java_home),
+ sudo = True,
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/hook.py
new file mode 100644
index 0000000..ce17776
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/hook.py
@@ -0,0 +1,37 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from shared_initialization import *
+from repo_initialization import *
+
+class BeforeInstallHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ self.run_custom_hook('before-ANY')
+ env.set_params(params)
+
+ install_repos()
+ install_packages()
+
+if __name__ == "__main__":
+ BeforeInstallHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/params.py
new file mode 100644
index 0000000..6193c11
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/params.py
@@ -0,0 +1,113 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.functions.version import format_stack_version, compare_versions
+from resource_management.core.system import System
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import default, format
+from resource_management.libraries.functions.expect import expect
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+sudo = AMBARI_SUDO_BINARY
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+#users and groups
+hbase_user = config['configurations']['hbase-env']['hbase_user']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+gmetad_user = config['configurations']['ganglia-env']["gmetad_user"]
+gmond_user = config['configurations']['ganglia-env']["gmond_user"]
+tez_user = config['configurations']['tez-env']["tez_user"]
+
+user_group = config['configurations']['cluster-env']['user_group']
+proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users")
+
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+
+# repo templates
+repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
+repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
+
+#hosts
+hostname = config["hostname"]
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+rm_host = default("/clusterHostInfo/rm_host", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
+hive_server_host = default("/clusterHostInfo/hive_server_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+hs_host = default("/clusterHostInfo/hs_host", [])
+jtnode_host = default("/clusterHostInfo/jtnode_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+storm_server_hosts = default("/clusterHostInfo/nimbus_hosts", [])
+falcon_host = default('/clusterHostInfo/falcon_server_hosts', [])
+
+has_sqoop_client = 'sqoop-env' in config['configurations']
+has_namenode = not len(namenode_host) == 0
+has_hs = not len(hs_host) == 0
+has_resourcemanager = not len(rm_host) == 0
+has_slaves = not len(slave_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_hcat_server_host = not len(hcat_server_hosts) == 0
+has_hive_server_host = not len(hive_server_host) == 0
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_zk_host = not len(zk_hosts) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_storm_server = not len(storm_server_hosts) == 0
+has_falcon_server = not len(falcon_host) == 0
+has_tez = 'tez-site' in config['configurations']
+
+is_namenode_master = hostname in namenode_host
+is_jtnode_master = hostname in jtnode_host
+is_rmnode_master = hostname in rm_host
+is_hsnode_master = hostname in hs_host
+is_hbase_master = hostname in hbase_master_hosts
+is_slave = hostname in slave_hosts
+if has_ganglia_server:
+ ganglia_server_host = ganglia_server_hosts[0]
+
+hbase_tmp_dir = "/tmp/hbase-hbase"
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#java params
+java_home = config['hostLevelParams']['java_home']
+artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
+jdk_name = default("/hostLevelParams/jdk_name", None) # None when jdk is already installed by user
+jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
+jce_location = config['hostLevelParams']['jdk_location']
+jdk_location = config['hostLevelParams']['jdk_location']
+ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False)
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
+if has_hbase_masters:
+ hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}")
+#repo params
+repo_info = config['hostLevelParams']['repo_info']
+service_repo_info = default("/hostLevelParams/service_repo_info",None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/repo_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/repo_initialization.py
new file mode 100644
index 0000000..a35dce7
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/repo_initialization.py
@@ -0,0 +1,68 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.resources.repository import Repository
+from resource_management.core.logger import Logger
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+
+# components_lits = repoName + postfix
+_UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+
+def _alter_repo(action, repo_string, repo_template):
+ """
+ @param action: "delete" or "create"
+ @param repo_string: e.g. "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+ """
+ repo_dicts = json.loads(repo_string)
+
+ if not isinstance(repo_dicts, list):
+ repo_dicts = [repo_dicts]
+
+ if 0 == len(repo_dicts):
+ Logger.info("Repository list is empty. Ambari may not be managing the repositories.")
+ else:
+ Logger.info("Initializing {0} repositories".format(str(len(repo_dicts))))
+
+ for repo in repo_dicts:
+ if not 'baseUrl' in repo:
+ repo['baseUrl'] = None
+ if not 'mirrorsList' in repo:
+ repo['mirrorsList'] = None
+
+ ubuntu_components = [ repo['repoName'] ] + _UBUNTU_REPO_COMPONENTS_POSTFIX
+
+ Repository(repo['repoId'],
+ action = action,
+ base_url = repo['baseUrl'],
+ mirror_list = repo['mirrorsList'],
+ repo_file_name = repo['repoName'],
+ repo_template = repo_template,
+ components = ubuntu_components, # ubuntu specific
+ )
+
+def install_repos():
+ import params
+ if params.host_sys_prepped:
+ return
+
+ template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu
+ _alter_repo("create", params.repo_info, template)
+ if params.service_repo_info:
+ _alter_repo("create", params.service_repo_info, template)
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/shared_initialization.py
new file mode 100644
index 0000000..1609050
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -0,0 +1,37 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions import stack_tools
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.core.resources.packaging import Package
+
+def install_packages():
+ import params
+ if params.host_sys_prepped:
+ return
+
+ packages = ['unzip', 'curl']
+ if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
+ stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
+ packages.append(stack_selector_package)
+ Package(packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-RESTART/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-RESTART/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-RESTART/scripts/hook.py
new file mode 100644
index 0000000..14b9d99
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-RESTART/scripts/hook.py
@@ -0,0 +1,29 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+class BeforeRestartHook(Hook):
+
+ def hook(self, env):
+ self.run_custom_hook('before-START')
+
+if __name__ == "__main__":
+ BeforeRestartHook().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/checkForFormat.sh
new file mode 100644
index 0000000..68aa96d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/checkForFormat.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+export hdfs_user=$1
+shift
+export conf_dir=$1
+shift
+export bin_dir=$1
+shift
+export mark_dir=$1
+shift
+export name_dirs=$*
+
+export EXIT_CODE=0
+export command="namenode -format"
+export list_of_non_empty_dirs=""
+
+mark_file=/var/run/hadoop/hdfs/namenode-formatted
+if [[ -f ${mark_file} ]] ; then
+ /var/lib/ambari-agent/ambari-sudo.sh rm -f ${mark_file}
+ /var/lib/ambari-agent/ambari-sudo.sh mkdir -p ${mark_dir}
+fi
+
+if [[ ! -d $mark_dir ]] ; then
+ for dir in `echo $name_dirs | tr ',' ' '` ; do
+ echo "NameNode Dirname = $dir"
+ cmd="ls $dir | wc -l | grep -q ^0$"
+ eval $cmd
+ if [[ $? -ne 0 ]] ; then
+ (( EXIT_CODE = $EXIT_CODE + 1 ))
+ list_of_non_empty_dirs="$list_of_non_empty_dirs $dir"
+ fi
+ done
+
+ if [[ $EXIT_CODE == 0 ]] ; then
+ /var/lib/ambari-agent/ambari-sudo.sh su ${hdfs_user} - -s /bin/bash -c "export PATH=$PATH:$bin_dir ; yes Y | hdfs --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
+ else
+ echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
+ fi
+else
+ echo "${mark_dir} exists. Namenode DFS already formatted"
+fi
+
+exit $EXIT_CODE
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/fast-hdfs-resource.jar
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/fast-hdfs-resource.jar b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/fast-hdfs-resource.jar
new file mode 100644
index 0000000..c90890b
Binary files /dev/null and b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/fast-hdfs-resource.jar differ
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/task-log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/task-log4j.properties b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/task-log4j.properties
new file mode 100644
index 0000000..7e12962
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/task-log4j.properties
@@ -0,0 +1,134 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+# Custom Logging levels
+
+hadoop.metrics.log.level=INFO
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+# Removes "deprecated" messages
+log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/topology_script.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/topology_script.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/topology_script.py
new file mode 100644
index 0000000..0f7a55c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/files/topology_script.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys, os
+from string import join
+import ConfigParser
+
+
+DEFAULT_RACK = "/default-rack"
+DATA_FILE_NAME = os.path.dirname(os.path.abspath(__file__)) + "/topology_mappings.data"
+SECTION_NAME = "network_topology"
+
+class TopologyScript():
+
+ def load_rack_map(self):
+ try:
+ #RACK_MAP contains both host name vs rack and ip vs rack mappings
+ mappings = ConfigParser.ConfigParser()
+ mappings.read(DATA_FILE_NAME)
+ return dict(mappings.items(SECTION_NAME))
+ except ConfigParser.NoSectionError:
+ return {}
+
+ def get_racks(self, rack_map, args):
+ if len(args) == 1:
+ return DEFAULT_RACK
+ else:
+ return join([self.lookup_by_hostname_or_ip(input_argument, rack_map) for input_argument in args[1:]],)
+
+ def lookup_by_hostname_or_ip(self, hostname_or_ip, rack_map):
+ #try looking up by hostname
+ rack = rack_map.get(hostname_or_ip)
+ if rack is not None:
+ return rack
+ #try looking up by ip
+ rack = rack_map.get(self.extract_ip(hostname_or_ip))
+ #try by localhost since hadoop could be passing in 127.0.0.1 which might not be mapped
+ return rack if rack is not None else rack_map.get("localhost.localdomain", DEFAULT_RACK)
+
+ #strips out port and slashes in case hadoop passes in something like 127.0.0.1/127.0.0.1:50010
+ def extract_ip(self, container_string):
+ return container_string.split("/")[0].split(":")[0]
+
+ def execute(self, args):
+ rack_map = self.load_rack_map()
+ rack = self.get_racks(rack_map, args)
+ print rack
+
+if __name__ == "__main__":
+ TopologyScript().execute(sys.argv)
http://git-wip-us.apache.org/repos/asf/ambari/blob/757267fc/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/hook.py
new file mode 100644
index 0000000..f21e4b1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-START/scripts/hook.py
@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from rack_awareness import create_topology_script_and_mapping
+from shared_initialization import setup_hadoop, setup_configs, create_javahome_symlink
+
+class BeforeStartHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ self.run_custom_hook('before-ANY')
+ env.set_params(params)
+
+ setup_hadoop()
+ setup_configs()
+ create_javahome_symlink()
+ create_topology_script_and_mapping()
+
+if __name__ == "__main__":
+ BeforeStartHook().execute()
[48/50] [abbrv] ambari git commit: AMBARI-19113. Add timeout for
LogSearch and Infra Solr metainfo.xml (oleewere)
Posted by nc...@apache.org.
AMBARI-19113. Add timeout for LogSearch and Infra Solr metainfo.xml (oleewere)
Change-Id: I8cd2e50def25febcc6c8d08a7f8f04625c69c7be
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/69cdaa77
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/69cdaa77
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/69cdaa77
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 69cdaa775f8c0990efa052c806d41db02c3acc52
Parents: 49d0895
Author: oleewere <ol...@gmail.com>
Authored: Wed Dec 7 12:20:45 2016 +0100
Committer: oleewere <ol...@gmail.com>
Committed: Wed Dec 7 12:28:59 2016 +0100
----------------------------------------------------------------------
.../resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml | 1 +
.../AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py | 3 ++-
.../main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml | 1 +
.../LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py | 3 ++-
.../src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py | 2 +-
.../src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py | 2 +-
6 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/69cdaa77/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml
index 8e1051b..b94812b 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/metainfo.xml
@@ -34,6 +34,7 @@
<commandScript>
<script>scripts/infra_solr.py</script>
<scriptType>PYTHON</scriptType>
+ <timeout>1800</timeout>
</commandScript>
<logs>
<log>
http://git-wip-us.apache.org/repos/asf/ambari/blob/69cdaa77/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
index 41cb504..5143cbb 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA/0.1.0/package/scripts/setup_infra_solr.py
@@ -112,4 +112,5 @@ def create_ambari_solr_znode():
solr_cloud_util.create_znode(
zookeeper_quorum=params.zookeeper_quorum,
solr_znode=params.infra_solr_znode,
- java64_home=params.java64_home)
\ No newline at end of file
+ java64_home=params.java64_home,
+ retry=30, interval=5)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/69cdaa77/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index 253294e..3b45bfe 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -36,6 +36,7 @@
<commandScript>
<script>scripts/logsearch.py</script>
<scriptType>PYTHON</scriptType>
+ <timeout>1800</timeout>
</commandScript>
<logs>
<log>
http://git-wip-us.apache.org/repos/asf/ambari/blob/69cdaa77/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 6c1a936..351126b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -136,4 +136,5 @@ def check_znode():
solr_cloud_util.check_znode(
zookeeper_quorum=params.zookeeper_quorum,
solr_znode=params.infra_solr_znode,
- java64_home=params.java64_home)
+ java64_home=params.java64_home,
+ retry=30, interval=5)
http://git-wip-us.apache.org/repos/asf/ambari/blob/69cdaa77/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py b/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
index 005283f..6af463c 100644
--- a/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
+++ b/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
@@ -95,7 +95,7 @@ class TestInfraSolr(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['infra-solr-log4j']['content'])
)
- self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --create-znode --retry 5 --interval 10')
+ self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --create-znode --retry 30 --interval 5')
self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --cluster-prop --property-name urlScheme --property-value http')
self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --setup-kerberos-plugin')
http://git-wip-us.apache.org/repos/asf/ambari/blob/69cdaa77/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index 6b4fab8..df54cd8 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -129,7 +129,7 @@ class TestLogSearch(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['logsearch-audit_logs-solrconfig']['content'])
)
- self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 5 --interval 10')
+ self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 30 --interval 5')
self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_hadoop_logs_0.[0-9]* --config-set hadoop_logs --retry 30 --interval 5')
self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_hadoop_logs_0.[0-9]*',
content=InlineTemplate(self.getConfig()['configurations']['logsearch-service_logs-solrconfig']['content']),
[04/50] [abbrv] ambari git commit: AMBARI-19070. Incorrect version
metadata for HDP 2.6 (smohanty)
Posted by nc...@apache.org.
AMBARI-19070. Incorrect version metadata for HDP 2.6 (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f6942f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f6942f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f6942f5
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 2f6942f5b6d09e07ed31e28ed70a741db899fbc7
Parents: 6c411ba
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Dec 2 18:48:09 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Dec 2 18:48:51 2016 -0800
----------------------------------------------------------------------
.../main/resources/stacks/HDP/2.6/services/ACCUMULO/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/FALCON/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/FLUME/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/HBASE/metainfo.xml | 2 +-
.../src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml | 2 +-
.../src/main/resources/stacks/HDP/2.6/services/HIVE/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/KAFKA/metainfo.xml | 2 +-
.../src/main/resources/stacks/HDP/2.6/services/KNOX/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/MAHOUT/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/OOZIE/metainfo.xml | 1 +
.../src/main/resources/stacks/HDP/2.6/services/PIG/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml | 2 +-
.../resources/stacks/HDP/2.6/services/RANGER_KMS/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/SLIDER/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/SPARK/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/SQOOP/metainfo.xml | 2 +-
.../main/resources/stacks/HDP/2.6/services/STORM/metainfo.xml | 2 +-
.../src/main/resources/stacks/HDP/2.6/services/TEZ/metainfo.xml | 2 +-
.../src/main/resources/stacks/HDP/2.6/services/YARN/metainfo.xml | 4 ++--
.../resources/stacks/HDP/2.6/services/ZOOKEEPER/metainfo.xml | 2 +-
21 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/ACCUMULO/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ACCUMULO/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ACCUMULO/metainfo.xml
index ec81f3e..3f619af 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ACCUMULO/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ACCUMULO/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>ACCUMULO</name>
- <version>1.7.0.2.5</version>
+ <version>1.7.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/FALCON/metainfo.xml
index 0b54385..b449388 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/FALCON/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/FALCON/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>FALCON</name>
- <version>0.10.0.2.5</version>
+ <version>0.10.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/FLUME/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/FLUME/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/FLUME/metainfo.xml
index 33ceb43..0d7cd1f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/FLUME/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/FLUME/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>FLUME</name>
- <version>1.5.2.2.5</version>
+ <version>1.5.2.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/metainfo.xml
index 0feaa5e..9bc3ee2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HBASE/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>HBASE</name>
- <version>1.1.2.2.5</version>
+ <version>1.1.2.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml
index a3e4a64..1fc7f51 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>HDFS</name>
- <version>2.7.1.2.5</version>
+ <version>2.7.3.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/metainfo.xml
index 3b43cc0..54c92d8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>HIVE</name>
- <version>1.2.1.2.5</version>
+ <version>1.2.1.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/metainfo.xml
index 12f6c45..047bbd2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/KAFKA/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>KAFKA</name>
- <version>0.10.0.2.5</version>
+ <version>0.10.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/KNOX/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/KNOX/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/KNOX/metainfo.xml
index 8f303a5..cd7b92c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/KNOX/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/KNOX/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>KNOX</name>
- <version>0.9.0.2.5</version>
+ <version>0.11.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/MAHOUT/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/MAHOUT/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/MAHOUT/metainfo.xml
index 9ed6957..3fe4d57 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/MAHOUT/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/MAHOUT/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>MAHOUT</name>
- <version>0.9.0.2.5</version>
+ <version>0.9.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/metainfo.xml
index 9e845c3..6392ed4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/metainfo.xml
@@ -20,6 +20,7 @@
<services>
<service>
<name>OOZIE</name>
+ <version>4.2.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/metainfo.xml
index 0dbf74c..266aa5e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>PIG</name>
- <version>0.16.0.2.5</version>
+ <version>0.16.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml
index 6287e00..a115134 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER/metainfo.xml
@@ -23,7 +23,7 @@
<services>
<service>
<name>RANGER</name>
- <version>0.6.0.2.5</version>
+ <version>0.7.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER_KMS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER_KMS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER_KMS/metainfo.xml
index ab065d9..1c0a2db 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER_KMS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER_KMS/metainfo.xml
@@ -23,7 +23,7 @@
<services>
<service>
<name>RANGER_KMS</name>
- <version>0.6.0.2.5</version>
+ <version>0.7.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/SLIDER/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SLIDER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SLIDER/metainfo.xml
index 6502bfe..0447906 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SLIDER/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SLIDER/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>SLIDER</name>
- <version>0.80.0.2.5</version>
+ <version>0.91.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/metainfo.xml
index d1129cd..21cbd0e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/metainfo.xml
@@ -23,7 +23,7 @@
<services>
<service>
<name>SPARK</name>
- <version>1.6.x.2.5</version>
+ <version>1.6.x.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml
index 2f83cf4..6440a3a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/metainfo.xml
@@ -23,7 +23,7 @@
<services>
<service>
<name>SPARK2</name>
- <version>2.0.x.2.5</version>
+ <version>2.0.x.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/SQOOP/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SQOOP/metainfo.xml
index eb67d63..2d1fe85 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SQOOP/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SQOOP/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>SQOOP</name>
- <version>1.4.6.2.5</version>
+ <version>1.4.6.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/metainfo.xml
index c1e9490..e5eb1c0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/STORM/metainfo.xml
@@ -21,7 +21,7 @@
<services>
<service>
<name>STORM</name>
- <version>1.0.1.2.5</version>
+ <version>1.0.1.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/metainfo.xml
index 30f0c25..457342a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>TEZ</name>
- <version>0.7.0.2.5</version>
+ <version>0.7.0.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/metainfo.xml
index 48b30a9..35b883e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/metainfo.xml
@@ -21,11 +21,11 @@
<services>
<service>
<name>YARN</name>
- <version>2.7.1.2.5</version>
+ <version>2.7.3.2.6</version>
</service>
<service>
<name>MAPREDUCE2</name>
- <version>2.7.1.2.5</version>
+ <version>2.7.3.2.6</version>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f6942f5/ambari-server/src/main/resources/stacks/HDP/2.6/services/ZOOKEEPER/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ZOOKEEPER/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ZOOKEEPER/metainfo.xml
index de8ebd9..e5f33f9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ZOOKEEPER/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ZOOKEEPER/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>ZOOKEEPER</name>
- <version>3.4.6.2.5</version>
+ <version>3.4.6.2.6</version>
</service>
</services>
</metainfo>
[08/50] [abbrv] ambari git commit: AMBARI-19007 Atlas to support
configuration of hooks from separate cluster (mugdha)
Posted by nc...@apache.org.
AMBARI-19007 Atlas to support configuration of hooks from separate cluster (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ad494e9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ad494e9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ad494e9
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8ad494e97302d5da69224c7ce9e493d5f20797c1
Parents: 12a3cd8
Author: Mugdha Varadkar <mu...@apache.org>
Authored: Fri Dec 2 18:15:39 2016 +0530
Committer: Mugdha Varadkar <mu...@apache.org>
Committed: Mon Dec 5 10:13:09 2016 +0530
----------------------------------------------------------------------
.../libraries/functions/setup_atlas_hook.py | 47 +++++-----
.../BlueprintConfigurationProcessor.java | 3 +-
.../server/upgrade/UpgradeCatalog250.java | 30 +++++++
.../0.5.0.2.1/configuration/falcon-env.xml | 17 ++++
.../FALCON/0.5.0.2.1/package/scripts/falcon.py | 2 +-
.../0.5.0.2.1/package/scripts/params_linux.py | 5 +-
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 17 ++++
.../HIVE/0.12.0.2.0/package/scripts/hcat.py | 2 +-
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 2 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 5 +-
.../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +-
.../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 17 ++++
.../1.4.4.2.0/configuration/sqoop-site.xml | 6 ++
.../1.4.4.2.0/package/scripts/params_linux.py | 5 +-
.../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 2 +-
.../STORM/0.10.0/configuration/storm-env.xml | 17 ++++
.../STORM/0.9.1/package/scripts/params_linux.py | 6 +-
.../STORM/0.9.1/package/scripts/storm.py | 2 +-
.../services/HIVE/configuration/hive-site.xml | 4 +
.../services/STORM/configuration/storm-site.xml | 4 +
.../stacks/HDP/2.3/services/stack_advisor.py | 57 +++++++++++-
.../stacks/HDP/2.5/services/stack_advisor.py | 6 --
.../BlueprintConfigurationProcessorTest.java | 11 +++
.../server/upgrade/UpgradeCatalog250Test.java | 93 ++++++++++++++++++++
.../stacks/2.3/common/test_stack_advisor.py | 28 +++++-
25 files changed, 339 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
index a1d2f95..eeae4bc 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
@@ -117,6 +117,8 @@ def setup_atlas_hook(service_name, service_props, atlas_hook_filepath, owner, gr
"""
import params
atlas_props = default('/configurations/application-properties', {})
+ merged_props = {}
+ merged_props.update(service_props)
if has_atlas_in_cluster():
# Take the subset
@@ -131,12 +133,12 @@ def setup_atlas_hook(service_name, service_props, atlas_hook_filepath, owner, gr
merged_props.update(service_props)
- Logger.info(format("Generating Atlas Hook config file {atlas_hook_filepath}"))
- PropertiesFile(atlas_hook_filepath,
- properties = merged_props,
- owner = owner,
- group = group,
- mode = 0644)
+ Logger.info(format("Generating Atlas Hook config file {atlas_hook_filepath}"))
+ PropertiesFile(atlas_hook_filepath,
+ properties = merged_props,
+ owner = owner,
+ group = group,
+ mode = 0644)
def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
@@ -157,22 +159,23 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
"""
import params
- if has_atlas_in_cluster():
- atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ \
- else format("{stack_root}/current/atlas-server")
-
- # Will only exist if this host contains Atlas Server
- atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name)
- if os.path.exists(atlas_hook_dir):
- Logger.info("Atlas Server is present on this host, will symlink jars inside of %s to %s if not already done." %
- (jar_source_dir, atlas_hook_dir))
-
- src_files = os.listdir(atlas_hook_dir)
- for file_name in src_files:
- atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name)
- source_lib_file_name = os.path.join(jar_source_dir, file_name)
- if os.path.isfile(atlas_hook_file_name):
- Link(source_lib_file_name, to=atlas_hook_file_name)
+ atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ \
+ else format("{stack_root}/current/atlas-server")
+
+ # Will only exist if this host contains Atlas Server
+ atlas_hook_dir = os.path.join(atlas_home_dir, "hook", hook_name)
+ if os.path.exists(atlas_hook_dir):
+ Logger.info("Atlas Server is present on this host, will symlink jars inside of %s to %s if not already done." %
+ (jar_source_dir, atlas_hook_dir))
+
+ src_files = os.listdir(atlas_hook_dir)
+ for file_name in src_files:
+ atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name)
+ source_lib_file_name = os.path.join(jar_source_dir, file_name)
+ if os.path.isfile(atlas_hook_file_name):
+ Link(source_lib_file_name, to=atlas_hook_file_name)
+ else:
+ Logger.info("Atlas hook directory path {0} doesn't exist".format(atlas_hook_dir))
def install_atlas_hook_packages(atlas_plugin_package, atlas_ubuntu_plugin_package, host_sys_prepped,
agent_stack_retry_on_unavailability, agent_stack_retry_count):
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index f890326..737e8de 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2523,9 +2523,10 @@ public class BlueprintConfigurationProcessor {
}
boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS");
+ boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook"));
// Append atlas hook if not already present.
- if (isAtlasInCluster) {
+ if (isAtlasInCluster || isAtlasHiveHookEnabled) {
if (!hiveHooksClean.contains(atlasHookClass)) {
hiveHooksClean.add(atlasHookClass);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 837f340..ff1663e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -137,6 +137,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
updateKafkaConfigs();
updateHiveLlapConfigs();
updateTablesForZeppelinViewRemoval();
+ updateAtlasConfigs();
}
protected void updateHostVersionTable() throws SQLException {
@@ -326,5 +327,34 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE,
new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false));
}
+
+ protected void updateAtlasConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ updateAtlasHookConfig(cluster, "HIVE", "hive-env", "hive.atlas.hook");
+ updateAtlasHookConfig(cluster, "STORM", "storm-env", "storm.atlas.hook");
+ updateAtlasHookConfig(cluster, "FALCON", "falcon-env", "falcon.atlas.hook");
+ updateAtlasHookConfig(cluster, "SQOOP", "sqoop-env", "sqoop.atlas.hook");
+ }
+ }
+ }
+ }
+
+ protected void updateAtlasHookConfig(Cluster cluster, String serviceName, String configType, String propertyName) throws AmbariException {
+ Set<String> installedServices = cluster.getServices().keySet();
+ if (installedServices.contains("ATLAS") && installedServices.contains(serviceName)) {
+ Config configEnv = cluster.getDesiredConfigByType(configType);
+ if (configEnv != null) {
+ Map<String, String> newProperties = new HashMap<>();
+ newProperties.put(propertyName, "true");
+ boolean updateProperty = configEnv.getProperties().containsKey(propertyName);
+ updateConfigurationPropertiesForCluster(cluster, configType, newProperties, updateProperty, true);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
index 09cced6..5663f57 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
@@ -21,6 +21,23 @@
-->
<configuration supports_adding_forbidden="true">
<property>
+ <name>falcon.atlas.hook</name>
+ <value>false</value>
+ <display-name>Enable Atlas Hook</display-name>
+ <description>Enable Atlas Hook</description>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.rest.address</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
<name>falcon_user</name>
<display-name>Falcon User</display-name>
<value>falcon</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 3ecb65b..78874ef 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -121,7 +121,7 @@ def falcon(type, action = None, upgrade_type=None):
cd_access = "a")
# Generate atlas-application.properties.xml file
- if params.falcon_atlas_support:
+ if params.falcon_atlas_support and params.enable_atlas_hook:
# If Atlas is added later than Falcon, this package will be absent.
install_atlas_hook_packages(params.atlas_plugin_package, params.atlas_ubuntu_plugin_package, params.host_sys_prepped,
params.agent_stack_retry_on_unavailability, params.agent_stack_retry_count)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 0fb21d0..6d1ae03 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -138,14 +138,15 @@ dfs_data_mirroring_dir = "/apps/data-mirroring"
########################################################
#region Atlas Hooks
falcon_atlas_application_properties = default('/configurations/falcon-atlas-application.properties', {})
+atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
+enable_atlas_hook = default('/configurations/falcon-env/falcon.atlas.hook', False)
# Calculate atlas_hook_cp to add to FALCON_EXTRA_CLASS_PATH
falcon_atlas_support = False
# Path to add to environment variable
atlas_hook_cp = ""
-if has_atlas_in_cluster():
- atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
+if enable_atlas_hook:
# stack_version doesn't contain a minor number of the stack (only first two numbers: 2.3). Get it from current_version_formatted
falcon_atlas_support = current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, current_version_formatted) \
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index a3ad66a..7a3390e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -21,6 +21,23 @@
-->
<configuration supports_adding_forbidden="true">
<property>
+ <name>hive.atlas.hook</name>
+ <value>false</value>
+ <display-name>Enable Atlas Hook</display-name>
+ <description>Enable Atlas Hook</description>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.rest.address</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
<name>hive.client.heapsize</name>
<value>512</value>
<description>Hive Client Java heap size</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py
index e952767..eecbcff 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat.py
@@ -82,6 +82,6 @@ def hcat():
)
# Generate atlas-application.properties.xml file
- if has_atlas_in_cluster():
+ if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index f08c458..ffc996c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -68,7 +68,7 @@ def hive(name=None):
mode=0644)
# Generate atlas-application.properties.xml file
- if has_atlas_in_cluster():
+ if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index a461bfe..f1a95ed 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -517,9 +517,8 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s
########################################################
#region Atlas Hooks
hive_atlas_application_properties = default('/configurations/hive-atlas-application.properties', {})
-
-if has_atlas_in_cluster():
- atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
+enable_atlas_hook = default('/configurations/hive-env/hive.atlas.hook', False)
+atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
#endregion
########################################################
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
index 00b057c..0af54af 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
@@ -133,7 +133,7 @@ def webhcat():
)
# Generate atlas-application.properties.xml file
- if has_atlas_in_cluster():
+ if params.enable_atlas_hook:
# WebHCat uses a different config dir than the rest of the daemons in Hive.
atlas_hook_filepath = os.path.join(params.config_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
index f682e97..508cfab 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
@@ -22,6 +22,23 @@
<configuration supports_adding_forbidden="true">
<!-- sqoop-env.sh -->
<property>
+ <name>sqoop.atlas.hook</name>
+ <value>false</value>
+ <display-name>Enable Atlas Hook</display-name>
+ <description>Enable Atlas Hook</description>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.rest.address</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
<name>content</name>
<display-name>sqoop-env template</display-name>
<description>This is the jinja template for sqoop-env.sh file</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
index f7823d2..5d87c4d 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
@@ -35,5 +35,11 @@
<property>
<name>sqoop.job.data.publish.class</name>
<on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>sqoop-env</type>
+ <name>sqoop.atlas.hook</name>
+ </property>
+ </depends-on>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
index 283f54d..c1138b3 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
@@ -130,7 +130,6 @@ jdk_location = config['hostLevelParams']['jdk_location']
########################################################
#region Atlas Hooks
sqoop_atlas_application_properties = default('/configurations/sqoop-atlas-application.properties', {})
-
-if has_atlas_in_cluster():
- atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
+enable_atlas_hook = default('/configurations/sqoop-env/sqoop.atlas.hook', False)
+atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
#endregion
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
index 68f06db..d005cbe 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
@@ -64,7 +64,7 @@ def sqoop(type=None):
)
# Generate atlas-application.properties.xml file and symlink the hook jars
- if has_atlas_in_cluster():
+ if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group)
setup_atlas_jar_symlinks("sqoop", params.sqoop_lib)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-env.xml b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-env.xml
index 9547335..3b814a9 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-env.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-env.xml
@@ -21,6 +21,23 @@
-->
<configuration supports_adding_forbidden="true">
<property>
+ <name>storm.atlas.hook</name>
+ <value>false</value>
+ <display-name>Enable Atlas Hook</display-name>
+ <description>Enable Atlas Hook</description>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.rest.address</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
<name>nimbus_seeds_supported</name>
<value>true</value>
<description/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index aca0681..4859534 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -214,10 +214,10 @@ jar_jvm_opts = ''
########################################################
#region Atlas Hooks
storm_atlas_application_properties = default('/configurations/storm-atlas-application.properties', {})
+enable_atlas_hook = default('/configurations/storm-env/storm.atlas.hook', False)
+atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
-if has_atlas_in_cluster():
- atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
-
+if enable_atlas_hook:
# Only append /etc/atlas/conf to classpath if on HDP 2.4.*
if check_stack_feature(StackFeature.ATLAS_CONF_DIR_IN_PATH, stack_version_formatted):
atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
index bda4fe2..f02ced4 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/storm.py
@@ -99,7 +99,7 @@ def storm(name=None):
)
# Generate atlas-application.properties.xml file and symlink the hook jars
- if has_atlas_in_cluster():
+ if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group)
storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib")
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index ab350dc..287db10 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -232,6 +232,10 @@ limitations under the License.
<depends-on>
<property>
<type>hive-env</type>
+ <name>hive.atlas.hook</name>
+ </property>
+ <property>
+ <type>hive-env</type>
<name>hive_timeline_logging_enabled</name>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml
index 2392079..5de6898 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml
@@ -67,6 +67,10 @@
<type>application-properties</type>
<name>atlas.authentication.method</name>
</property>
+ <property>
+ <type>storm-env</type>
+ <name>storm.atlas.hook</name>
+ </property>
</depends-on>
<on-ambari-upgrade add="false"/>
<on-stack-upgrade merge="false"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 4e2a994..5d34779 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -81,7 +81,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"RANGER": self.recommendRangerConfigurations,
"RANGER_KMS": self.recommendRangerKMSConfigurations,
"STORM": self.recommendStormConfigurations,
- "SQOOP": self.recommendSqoopConfigurations
+ "SQOOP": self.recommendSqoopConfigurations,
+ "FALCON": self.recommendFalconConfigurations
}
parentRecommendConfDict.update(childRecommendConfDict)
return parentRecommendConfDict
@@ -219,6 +220,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
super(HDP23StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts)
putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services)
+ putHiveEnvProperty = self.putProperty(configurations, "hive-env", services)
putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
# hive_security_authorization == 'ranger'
@@ -258,7 +260,19 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
hive_hooks = [x for x in hive_hooks if x != ""]
is_atlas_present_in_cluster = "ATLAS" in servicesList
+ enable_atlas_hook = False
if is_atlas_present_in_cluster:
+ putHiveEnvProperty("hive.atlas.hook", "true")
+ else:
+ putHiveEnvProperty("hive.atlas.hook", "false")
+
+ if ('hive-env' in services['configurations']) and ('hive.atlas.hook' in services['configurations']['hive-env']['properties']):
+ if 'hive-env' in configurations and 'hive.atlas.hook' in configurations['hive-env']['properties']:
+ enable_atlas_hook = configurations['hive-env']['properties']['hive.atlas.hook'] == "true"
+ elif 'hive-env' in services['configurations'] and 'hive.atlas.hook' in services['configurations']['hive-env']['properties']:
+ enable_atlas_hook = services['configurations']['hive-env']['properties']['hive.atlas.hook'] == "true"
+
+ if enable_atlas_hook:
# Append atlas hook if not already present.
is_atlas_hook_in_config = atlas_hook_class in hive_hooks
if not is_atlas_hook_in_config:
@@ -791,14 +805,28 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts):
putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services)
+ putSqoopEnvProperty = self.putProperty(configurations, "sqoop-env", services)
+ enable_atlas_hook = False
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if "ATLAS" in servicesList:
+ putSqoopEnvProperty("sqoop.atlas.hook", "true")
+ else:
+ putSqoopEnvProperty("sqoop.atlas.hook", "false")
+
+ if ('sqoop-env' in services['configurations']) and ('sqoop.atlas.hook' in services['configurations']['sqoop-env']['properties']):
+ if 'sqoop-env' in configurations and 'sqoop.atlas.hook' in configurations['sqoop-env']['properties']:
+ enable_atlas_hook = configurations['sqoop-env']['properties']['sqoop.atlas.hook'] == "true"
+ elif 'sqoop-env' in services['configurations'] and 'sqoop.atlas.hook' in services['configurations']['sqoop-env']['properties']:
+ enable_atlas_hook = services['configurations']['sqoop-env']['properties']['sqoop.atlas.hook'] == "true"
+
+ if enable_atlas_hook:
putSqoopSiteProperty('sqoop.job.data.publish.class', 'org.apache.atlas.sqoop.hook.SqoopHook')
def recommendStormConfigurations(self, configurations, clusterData, services, hosts):
super(HDP23StackAdvisor, self).recommendStormConfigurations(configurations, clusterData, services, hosts)
putStormStartupProperty = self.putProperty(configurations, "storm-site", services)
+ putStormEnvProperty = self.putProperty(configurations, "storm-env", services)
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if "storm-site" in services["configurations"]:
@@ -814,11 +842,23 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
atlas_is_present = "ATLAS" in servicesList
atlas_hook_class = "org.apache.atlas.storm.hook.StormAtlasHook"
atlas_hook_is_set = atlas_hook_class in notifier_plugin_value
+ enable_atlas_hook = False
+
+ if atlas_is_present:
+ putStormEnvProperty("storm.atlas.hook", "true")
+ else:
+ putStormEnvProperty("storm.atlas.hook", "false")
- if atlas_is_present and not atlas_hook_is_set:
+ if ('storm-env' in services['configurations']) and ('storm.atlas.hook' in services['configurations']['storm-env']['properties']):
+ if 'storm-env' in configurations and 'storm.atlas.hook' in configurations['storm-env']['properties']:
+ enable_atlas_hook = configurations['storm-env']['properties']['storm.atlas.hook'] == "true"
+ elif 'storm-env' in services['configurations'] and 'storm.atlas.hook' in services['configurations']['storm-env']['properties']:
+ enable_atlas_hook = services['configurations']['storm-env']['properties']['storm.atlas.hook'] == "true"
+
+ if enable_atlas_hook and not atlas_hook_is_set:
notifier_plugin_value = atlas_hook_class if notifier_plugin_value == " " else ",".join([notifier_plugin_value, atlas_hook_class])
- if not atlas_is_present and atlas_hook_is_set:
+ if not enable_atlas_hook and atlas_hook_is_set:
application_classes = [item for item in notifier_plugin_value.split(",") if item != atlas_hook_class and item != " "]
notifier_plugin_value = ",".join(application_classes) if application_classes else " "
@@ -828,6 +868,17 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
putStormStartupPropertyAttribute = self.putPropertyAttribute(configurations, "storm-site")
putStormStartupPropertyAttribute(notifier_plugin_property, 'delete', 'true')
+ def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
+
+ putFalconEnvProperty = self.putProperty(configurations, "falcon-env", services)
+ enable_atlas_hook = False
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+ if "ATLAS" in servicesList:
+ putFalconEnvProperty("falcon.atlas.hook", "true")
+ else:
+ putFalconEnvProperty("falcon.atlas.hook", "false")
+
def getServiceConfigurationValidators(self):
parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
childValidators = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 923c1f7..57555ee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -84,12 +84,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"})
putOozieEnvProperty("oozie_admin_users", newAdminUsers)
- def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
- # Set the classname for the Falcon hook to 2.5 classname = all other 2.3
- # stack advisor changes are needed.
- self.atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService"
- super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts)
-
def createComponentLayoutRecommendations(self, services, hosts):
parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 5bedb9d..0ad0351 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -5923,6 +5923,10 @@ public class BlueprintConfigurationProcessorTest {
hiveProperties.put("hive.exec.post.hooks", "");
properties.put("hive-site", hiveProperties);
+ Map<String, String> hiveEnv = new HashMap<String, String>();
+ hiveEnv.put("hive.atlas.hook", "false");
+ properties.put("hive-env", hiveEnv);
+
Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
Configuration parentClusterConfig = new Configuration(parentProperties,
@@ -6000,6 +6004,9 @@ public class BlueprintConfigurationProcessorTest {
hiveProperties.put("hive.exec.post.hooks", "");
properties.put("hive-site", hiveProperties);
+ Map<String, String> hiveEnv = new HashMap<String, String>();
+ properties.put("hive-env", hiveEnv);
+
return properties;
}
@@ -6051,6 +6058,10 @@ public class BlueprintConfigurationProcessorTest {
hiveProperties.put("hive.exec.post.hooks", "foo");
properties.put("hive-site", hiveProperties);
+ Map<String, String> hiveEnv = new HashMap<String, String>();
+ hiveEnv.put("hive.atlas.hook", "false");
+ properties.put("hive-env", hiveEnv);
+
Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
Configuration parentClusterConfig = new Configuration(parentProperties,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 0b2b32d..16da70c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -53,6 +53,7 @@ import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.stack.OsFamily;
import org.easymock.Capture;
import org.easymock.EasyMock;
@@ -214,6 +215,7 @@ public class UpgradeCatalog250Test {
Method updateHiveLlapConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHiveLlapConfigs");
Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
Method updateTablesForZeppelinViewRemoval = UpgradeCatalog250.class.getDeclaredMethod("updateTablesForZeppelinViewRemoval");
+ Method updateAtlasConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAtlasConfigs");
UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class)
.addMockedMethod(updateAmsConfigs)
@@ -221,6 +223,7 @@ public class UpgradeCatalog250Test {
.addMockedMethod(updateHiveLlapConfigs)
.addMockedMethod(addNewConfigurationsFromXml)
.addMockedMethod(updateTablesForZeppelinViewRemoval)
+ .addMockedMethod(updateAtlasConfigs)
.createMock();
upgradeCatalog250.updateAMSConfigs();
@@ -238,6 +241,9 @@ public class UpgradeCatalog250Test {
upgradeCatalog250.updateTablesForZeppelinViewRemoval();
expectLastCall().once();
+ upgradeCatalog250.updateAtlasConfigs();
+ expectLastCall().once();
+
replay(upgradeCatalog250);
upgradeCatalog250.executeDMLUpdates();
@@ -372,4 +378,91 @@ public class UpgradeCatalog250Test {
Map<String, String> updatedProperties = propertiesCapture.getValue();
assertTrue(Maps.difference(newProperties, updatedProperties).areEqual());
}
+
+ @Test
+ public void testUpdateAtlasConfigs() throws Exception {
+
+ Map<String, String> oldHiveProperties = new HashMap<String, String>();
+ Map<String, String> newHiveProperties = new HashMap<String, String>();
+
+ oldHiveProperties.put("hive.atlas.hook", "false");
+ newHiveProperties.put("hive.atlas.hook", "true");
+ testUpdateAtlasHookConfig(oldHiveProperties, newHiveProperties, "hive-env");
+
+ Map<String, String> oldStormProperties = new HashMap<String, String>();
+ Map<String, String> newStormProperties = new HashMap<String, String>();
+ oldStormProperties.put("storm.atlas.hook", "false");
+ newStormProperties.put("storm.atlas.hook", "true");
+ testUpdateAtlasHookConfig(oldStormProperties, newStormProperties, "storm-env");
+
+ Map<String, String> oldFalconProperties = new HashMap<String, String>();
+ Map<String, String> newFalconProperties = new HashMap<String, String>();
+ oldFalconProperties.put("falcon.atlas.hook", "false");
+ newFalconProperties.put("falcon.atlas.hook", "true");
+ testUpdateAtlasHookConfig(oldFalconProperties, newFalconProperties, "falcon-env");
+
+ Map<String, String> oldSqoopProperties = new HashMap<String, String>();
+ Map<String, String> newSqoopProperties = new HashMap<String, String>();
+ oldSqoopProperties.put("sqoop.atlas.hook", "false");
+ newSqoopProperties.put("sqoop.atlas.hook", "true");
+ testUpdateAtlasHookConfig(oldSqoopProperties, newSqoopProperties, "sqoop-env");
+ }
+
+ public void testUpdateAtlasHookConfig(Map<String, String> oldProperties, Map<String, String> newProperties, String configType) throws Exception {
+
+ Map<String, Service> installedServices = new HashMap<String, Service>() {
+ {
+ put("ATLAS", null);
+ put("HIVE", null);
+ put("STORM", null);
+ put("FALCON", null);
+ put("SQOOP", null);
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ final Service service = createStrictMock(Service.class);
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getClusterName()).andReturn("cl1").once();
+ expect(cluster.getServices()).andReturn(installedServices).atLeastOnce();
+
+ Config mockAtlasConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType(configType)).andReturn(mockAtlasConfig).atLeastOnce();
+ expect(mockAtlasConfig.getProperties()).andReturn(oldProperties).anyTimes();
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+ expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes();
+
+ replay(injector, clusters, mockAtlasConfig, cluster);
+
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("createConfiguration")
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+ .createNiceMock();
+
+ Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+ Capture<Map> propertiesCapture = EasyMock.newCapture();
+
+ expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector2);
+ new UpgradeCatalog250(injector2).updateAtlasConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedProperties = propertiesCapture.getValue();
+ assertTrue(Maps.difference(newProperties, updatedProperties).areEqual());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ad494e9/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 39cbbce..b5ff24c 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -587,7 +587,8 @@ class TestHDP23StackAdvisor(TestCase):
'hive_exec_orc_storage_strategy': 'SPEED',
'hive_security_authorization': 'None',
'hive_timeline_logging_enabled': 'true',
- 'hive_txn_acid': 'off'
+ 'hive_txn_acid': 'off',
+ 'hive.atlas.hook': 'false'
}
},
'hive-site': {
@@ -745,6 +746,7 @@ class TestHDP23StackAdvisor(TestCase):
},
"hive-env": {
"properties": {
+ "hive.atlas.hook": "false"
}
},
"hive-site": {
@@ -863,7 +865,8 @@ class TestHDP23StackAdvisor(TestCase):
'hive_exec_orc_storage_strategy': 'SPEED',
'hive_security_authorization': 'None',
'hive_timeline_logging_enabled': 'true',
- 'hive_txn_acid': 'off'
+ 'hive_txn_acid': 'off',
+ 'hive.atlas.hook': 'true'
}
},
'hive-site': {
@@ -1045,6 +1048,7 @@ class TestHDP23StackAdvisor(TestCase):
},
"hive-env": {
"properties": {
+ "hive.atlas.hook": "false"
}
},
"hive-site": {
@@ -1733,6 +1737,11 @@ class TestHDP23StackAdvisor(TestCase):
"properties": {
"ranger-storm-plugin-enabled": "No"
}
+ },
+ "storm-env": {
+ "properties": {
+ "storm.atlas.hook": "true"
+ }
}
}
services = {
@@ -1773,6 +1782,11 @@ class TestHDP23StackAdvisor(TestCase):
"properties": {
"ranger-storm-plugin-enabled": "No"
}
+ },
+ "storm-env": {
+ "properties": {
+ "storm.atlas.hook": "false"
+ }
}
},
"changed-configurations": [ ]
@@ -1830,6 +1844,11 @@ class TestHDP23StackAdvisor(TestCase):
'properties': {
'sqoop.job.data.publish.class': 'org.apache.atlas.sqoop.hook.SqoopHook',
}
+ },
+ 'sqoop-env': {
+ 'properties': {
+ 'sqoop.atlas.hook': 'true'
+ }
}
}
services = {
@@ -1864,6 +1883,11 @@ class TestHDP23StackAdvisor(TestCase):
"properties": {
"sqoop.job.data.publish.class": "foo"
}
+ },
+ "sqoop-env": {
+ "properties": {
+ "sqoop.atlas.hook": "false"
+ }
}
},
"changed-configurations": [ ]
[31/50] [abbrv] ambari git commit: AMBARI-19106. Kerberos wizard is
broken if stacks doesn't contain Ranger (alexantonenko)
Posted by nc...@apache.org.
AMBARI-19106. Kerberos wizard is broken if stacks doesn't contain Ranger (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9b46fd2a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9b46fd2a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9b46fd2a
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 9b46fd2acd6a5d1bd8ce474315fb318f1cfe6a3a
Parents: 0f29751
Author: Alex Antonenko <hi...@gmail.com>
Authored: Tue Dec 6 19:18:34 2016 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Tue Dec 6 19:18:34 2016 +0200
----------------------------------------------------------------------
ambari-web/app/views/common/controls_view.js | 3 +-
.../test/views/common/controls_view_test.js | 74 +++++++++++++++++++-
2 files changed, 75 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9b46fd2a/ambari-web/app/views/common/controls_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/controls_view.js b/ambari-web/app/views/common/controls_view.js
index dd475b6..bcb90d8 100644
--- a/ambari-web/app/views/common/controls_view.js
+++ b/ambari-web/app/views/common/controls_view.js
@@ -1048,11 +1048,12 @@ App.CheckDBConnectionView = Ember.View.extend({
isBtnDisabled: Em.computed.or('!isValidationPassed', 'isConnecting'),
/** @property {object} requiredProperties - properties that necessary for database connection **/
requiredProperties: function() {
+ var ranger = App.StackService.find().findProperty('serviceName', 'RANGER');
var propertiesMap = {
OOZIE: ['oozie.db.schema.name', 'oozie.service.JPAService.jdbc.username', 'oozie.service.JPAService.jdbc.password', 'oozie.service.JPAService.jdbc.driver', 'oozie.service.JPAService.jdbc.url'],
HIVE: ['ambari.hive.db.schema.name', 'javax.jdo.option.ConnectionUserName', 'javax.jdo.option.ConnectionPassword', 'javax.jdo.option.ConnectionDriverName', 'javax.jdo.option.ConnectionURL'],
KERBEROS: ['kdc_hosts'],
- RANGER: App.StackService.find('RANGER').compareCurrentVersion('0.5') > -1 ?
+ RANGER: ranger && App.StackService.find().findProperty('serviceName', 'RANGER').compareCurrentVersion('0.5') > -1 ?
['db_user', 'db_password', 'db_name', 'ranger.jpa.jdbc.url', 'ranger.jpa.jdbc.driver'] :
['db_user', 'db_password', 'db_name', 'ranger_jdbc_connection_url', 'ranger_jdbc_driver']
};
http://git-wip-us.apache.org/repos/asf/ambari/blob/9b46fd2a/ambari-web/test/views/common/controls_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/common/controls_view_test.js b/ambari-web/test/views/common/controls_view_test.js
index c5c5338..3ff5253 100644
--- a/ambari-web/test/views/common/controls_view_test.js
+++ b/ambari-web/test/views/common/controls_view_test.js
@@ -651,7 +651,6 @@ describe('App.CheckDBConnectionView', function () {
expect(view.get('masterHostName')).to.equal(item.value);
});
});
-
});
describe('#setResponseStatus', function () {
@@ -836,6 +835,79 @@ describe('App.CheckDBConnectionView', function () {
expect(args[0]).exists;
});
});
+
+ describe('#requriedProperties', function() {
+ var cases;
+ beforeEach(function() {
+ this.stackServiceStub = sinon.stub(App.StackService, 'find');
+ });
+ afterEach(function() {
+ this.stackServiceStub.restore();
+ });
+
+ cases = [
+ {
+ stackServices: [
+ {name: 'OOZIE', version: '1.0.0'}
+ ],
+ parentViewServiceName: 'OOZIE',
+ e: ['oozie.db.schema.name', 'oozie.service.JPAService.jdbc.username', 'oozie.service.JPAService.jdbc.password', 'oozie.service.JPAService.jdbc.driver', 'oozie.service.JPAService.jdbc.url'],
+ m: 'should return Oozie specific properties'
+ },
+ {
+ stackServices: [
+ {name: 'HIVE', version: '1.0.0'}
+ ],
+ parentViewServiceName: 'HIVE',
+ e: ['ambari.hive.db.schema.name', 'javax.jdo.option.ConnectionUserName', 'javax.jdo.option.ConnectionPassword', 'javax.jdo.option.ConnectionDriverName', 'javax.jdo.option.ConnectionURL'],
+ m: 'should return Hive specific properties'
+ },
+ {
+ stackServices: [
+ {name: 'KERBEROS', version: '1.0.0'}
+ ],
+ parentViewServiceName: 'KERBEROS',
+ e: ['kdc_hosts'],
+ m: 'should return specific Kerberos specific properties'
+ },
+ {
+ stackServices: [
+ {name: 'RANGER', version: '0.4.9'}
+ ],
+ parentViewServiceName: 'RANGER',
+ e: ['db_user', 'db_password', 'db_name', 'ranger_jdbc_connection_url', 'ranger_jdbc_driver'],
+ m: 'should return specific properties for Ranger when its version < 0.5'
+ },
+ {
+ stackServices: [
+ {name: 'RANGER', version: '1.0.0'}
+ ],
+ parentViewServiceName: 'RANGER',
+ e: ['db_user', 'db_password', 'db_name', 'ranger.jpa.jdbc.url', 'ranger.jpa.jdbc.driver'],
+ m: 'should return specific properties for Ranger when its version > 0.5'
+ }
+ ];
+
+ cases.forEach(function(test) {
+ it(test.m, function() {
+ this.stackServiceStub.returns(test.stackServices.map(function(service) {
+ return Em.Object.create({
+ serviceName: service.name,
+ serviceVersion: service.version,
+ compareCurrentVersion: App.StackService.proto().compareCurrentVersion
+ });
+ }));
+ var view = App.CheckDBConnectionView.create({
+ parentView: {
+ service: {
+ serviceName: test.parentViewServiceName
+ }
+ }
+ });
+ expect(view.get('requiredProperties')).to.be.eql(test.e);
+ });
+ });
+ });
});
describe('App.BaseUrlTextField', function () {
[28/50] [abbrv] ambari git commit: AMBARI-19080 Ambari stack-advisor
error while adding knox service (mugdha)
Posted by nc...@apache.org.
AMBARI-19080 Ambari stack-advisor error while adding knox service (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/149104c2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/149104c2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/149104c2
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 149104c2713f320116fc63daea0260f3eafca61b
Parents: a2f7a030
Author: Mugdha Varadkar <mu...@apache.org>
Authored: Tue Dec 6 14:13:59 2016 +0530
Committer: Mugdha Varadkar <mu...@apache.org>
Committed: Tue Dec 6 18:34:48 2016 +0530
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 4 ++-
.../stacks/2.3/common/test_stack_advisor.py | 29 +++++++++++++++++++-
2 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/149104c2/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 5d34779..b6c83bd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -768,7 +768,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
knox_port = '8443'
if 'KNOX' in servicesList:
knox_hosts = self.getComponentHostNames(services, "KNOX", "KNOX_GATEWAY")
- knox_host = knox_hosts[0]
+ if len(knox_hosts) > 0:
+ knox_hosts.sort()
+ knox_host = knox_hosts[0]
if 'gateway-site' in services['configurations'] and 'gateway.port' in services['configurations']["gateway-site"]["properties"]:
knox_port = services['configurations']["gateway-site"]["properties"]['gateway.port']
putRangerAdminProperty('ranger.sso.providerurl', 'https://{0}:{1}/gateway/knoxsso/api/v1/websso'.format(knox_host, knox_port))
http://git-wip-us.apache.org/repos/asf/ambari/blob/149104c2/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index b5ff24c..4a77086 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1466,6 +1466,31 @@ class TestHDP23StackAdvisor(TestCase):
}
]
},
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.3/services/KNOX",
+ "StackServices": {
+ "service_name": "KNOX",
+ "service_version": "0.9.0.2.3",
+ "stack_name": "HDP",
+ "stack_version": "2.3"
+ },
+ "components": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.3/services/KNOX/components/KNOX_GATEWAY",
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1+",
+ "component_category": "MASTER",
+ "component_name": "KNOX_GATEWAY",
+ "display_name": "Knox Gateway",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ "dependencies": []
+ }
+ ]
+ }
],
"configurations": {
"admin-properties": {
@@ -1477,6 +1502,7 @@ class TestHDP23StackAdvisor(TestCase):
"properties": {
"ranger.service.http.port": "7777",
"ranger.service.http.enabled": "true",
+ "ranger.sso.providerurl": "",
}
}
},
@@ -1519,7 +1545,8 @@ class TestHDP23StackAdvisor(TestCase):
'ranger-admin-site': {
'properties': {
"ranger.audit.solr.zookeepers": "NONE",
- "ranger.audit.source.type": "solr"
+ "ranger.audit.source.type": "solr",
+ "ranger.sso.providerurl": "https://c6401.ambari.apache.org:8443/gateway/knoxsso/api/v1/websso"
}
},
'ranger-env': {
[41/50] [abbrv] ambari git commit: AMBARI-19094. HDP 3.0 support for
YARN/MR with configs, kerberos, widgets, metrics, quicklinks,
and themes (alejandro)
Posted by nc...@apache.org.
AMBARI-19094. HDP 3.0 support for YARN/MR with configs, kerberos, widgets, metrics, quicklinks, and themes (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a62119ab
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a62119ab
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a62119ab
Branch: refs/heads/branch-dev-patch-upgrade
Commit: a62119ab9f1abc355bc99ee0fc71afac6723d827
Parents: 2a710b3
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Tue Dec 6 16:04:56 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Tue Dec 6 16:54:11 2016 -0800
----------------------------------------------------------------------
.../HDP/3.0/services/YARN/YARN_widgets.json | 670 +++++++++++++++
.../YARN/configuration-mapred/mapred-env.xml | 51 ++
.../YARN/configuration-mapred/mapred-site.xml | 134 +++
.../YARN/configuration/capacity-scheduler.xml | 71 ++
.../YARN/configuration/ranger-yarn-audit.xml | 177 ++++
.../ranger-yarn-plugin-properties.xml | 82 ++
.../configuration/ranger-yarn-policymgr-ssl.xml | 66 ++
.../YARN/configuration/ranger-yarn-security.xml | 58 ++
.../services/YARN/configuration/yarn-env.xml | 200 +++++
.../services/YARN/configuration/yarn-log4j.xml | 103 +++
.../services/YARN/configuration/yarn-site.xml | 814 +++++++++++++++++++
.../stacks/HDP/3.0/services/YARN/kerberos.json | 278 +++++++
.../stacks/HDP/3.0/services/YARN/metainfo.xml | 173 ++++
.../YARN/quicklinks-mapred/quicklinks.json | 80 ++
.../services/YARN/quicklinks/quicklinks.json | 80 ++
.../3.0/services/YARN/themes-mapred/theme.json | 132 +++
.../HDP/3.0/services/YARN/themes/theme.json | 250 ++++++
17 files changed, 3419 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/YARN_widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/YARN_widgets.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/YARN_widgets.json
new file mode 100644
index 0000000..782f21d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/YARN_widgets.json
@@ -0,0 +1,670 @@
+{
+ "layouts": [
+ {
+ "layout_name": "default_yarn_dashboard",
+ "display_name": "Standard YARN Dashboard",
+ "section_name": "YARN_SUMMARY",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "Memory Utilization",
+ "description": "Percentage of total memory allocated to containers running in the cluster.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AllocatedMB",
+ "metric_path": "metrics/yarn/Queue/root/AllocatedMB",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AvailableMB",
+ "metric_path": "metrics/yarn/Queue/root/AvailableMB",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ }
+ ],
+ "values": [
+ {
+ "name": "Memory Utilization",
+ "value": "${(yarn.QueueMetrics.Queue=root.AllocatedMB / (yarn.QueueMetrics.Queue=root.AllocatedMB + yarn.QueueMetrics.Queue=root.AvailableMB)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "CPU Utilization",
+ "description": "Percentage of total virtual cores allocated to containers running in the cluster.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AllocatedVCores",
+ "metric_path": "metrics/yarn/Queue/root/AllocatedVCores",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AvailableVCores",
+ "metric_path": "metrics/yarn/Queue/root/AvailableVCores",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ }
+ ],
+ "values": [
+ {
+ "name": "Total Allocatable CPU Utilized across NodeManager",
+ "value": "${(yarn.QueueMetrics.Queue=root.AllocatedVCores / (yarn.QueueMetrics.Queue=root.AllocatedVCores + yarn.QueueMetrics.Queue=root.AvailableVCores)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Bad Local Disks",
+ "description": "Number of unhealthy local disks across all NodeManagers.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.BadLocalDirs",
+ "metric_path": "metrics/yarn/BadLocalDirs",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.BadLogDirs",
+ "metric_path": "metrics/yarn/BadLogDirs",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Number of unhealthy local disks for NodeManager",
+ "value": "${yarn.NodeManagerMetrics.BadLocalDirs + yarn.NodeManagerMetrics.BadLogDirs}"
+ }
+ ],
+ "properties": {
+ "display_unit": ""
+ }
+ },
+ {
+ "widget_name": "Container Failures",
+ "description": "Percentage of all containers failing in the cluster.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersFailed._rate",
+ "metric_path": "metrics/yarn/ContainersFailed._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersCompleted._rate",
+ "metric_path": "metrics/yarn/ContainersCompleted._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersLaunched._rate",
+ "metric_path": "metrics/yarn/ContainersLaunched._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersIniting._sum",
+ "metric_path": "metrics/yarn/ContainersIniting._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersKilled._rate",
+ "metric_path": "metrics/yarn/ContainersKilled._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersRunning._sum",
+ "metric_path": "metrics/yarn/ContainersRunning._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Container Failures",
+ "value": "${(yarn.NodeManagerMetrics.ContainersFailed._rate/(yarn.NodeManagerMetrics.ContainersFailed._rate + yarn.NodeManagerMetrics.ContainersCompleted._rate + yarn.NodeManagerMetrics.ContainersLaunched._rate + yarn.NodeManagerMetrics.ContainersIniting._sum + yarn.NodeManagerMetrics.ContainersKilled._rate + yarn.NodeManagerMetrics.ContainersRunning._sum)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "App Failures",
+ "description": "Percentage of all launched applications failing in the cluster.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsFailed._rate",
+ "metric_path": "metrics/yarn/Queue/root/AppsFailed._rate",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsKilled._rate",
+ "metric_path": "metrics/yarn/Queue/root/AppsKilled._rate",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsPending",
+ "metric_path": "metrics/yarn/Queue/root/AppsPending",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsRunning",
+ "metric_path": "metrics/yarn/Queue/root/AppsRunning",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsSubmitted._rate",
+ "metric_path": "metrics/yarn/Queue/root/AppsSubmitted._rate",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ },
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsCompleted._rate",
+ "metric_path": "metrics/yarn/Queue/root/AppsCompleted._rate",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ }
+ ],
+ "values": [
+ {
+ "name": "App Failures",
+ "value": "${(yarn.QueueMetrics.Queue=root.AppsFailed._rate/(yarn.QueueMetrics.Queue=root.AppsFailed._rate + yarn.QueueMetrics.Queue=root.AppsKilled._rate + yarn.QueueMetrics.Queue=root.AppsPending + yarn.QueueMetrics.Queue=root.AppsRunning + yarn.QueueMetrics.Queue=root.AppsSubmitted._rate + yarn.QueueMetrics.Queue=root.AppsCompleted._rate)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Pending Apps",
+ "description": "Count of applications waiting for cluster resources to become available.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.QueueMetrics.Queue=root.AppsPending",
+ "metric_path": "metrics/yarn/Queue/root/AppsPending",
+ "service_name": "YARN",
+ "component_name": "RESOURCEMANAGER",
+ "host_component_criteria": "host_components/HostRoles/ha_state=ACTIVE"
+ }
+ ],
+ "values": [
+ {
+ "name": "Pending Apps",
+ "value": "${yarn.QueueMetrics.Queue=root.AppsPending}"
+ }
+ ],
+ "properties": {
+ "display_unit": "Apps",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Cluster Memory",
+ "description": "Percentage of memory used across all NodeManager hosts.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "mem_total._sum",
+ "metric_path": "metrics/memory/mem_total._avg",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "mem_free._sum",
+ "metric_path": "metrics/memory/mem_free._avg",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Memory utilization",
+ "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Cluster Disk",
+ "description": "Sum of disk throughput for all NodeManager hosts.",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "read_bps._sum",
+ "metric_path": "metrics/disk/read_bps._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "write_bps._sum",
+ "metric_path": "metrics/disk/write_bps._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Read throughput",
+ "value": "${read_bps._sum/1048576}"
+ },
+ {
+ "name": "Write throughput",
+ "value": "${write_bps._sum/1048576}"
+ }
+ ],
+ "properties": {
+ "display_unit": "Mbps",
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Cluster Network",
+ "description": "Average of Network utilized across all NodeManager hosts.",
+ "default_section_name": "YARN_SUMMARY",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "pkts_in._avg",
+ "metric_path": "metrics/network/pkts_in._avg",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "pkts_out._avg",
+ "metric_path": "metrics/network/pkts_out._avg",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Packets In",
+ "value": "${pkts_in._avg}"
+ },
+ {
+ "name": "Packets Out",
+ "value": "${pkts_out._avg}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1"
+ }
+ },
+ {
+ "widget_name": "Cluster CPU",
+ "description": "Percentage of CPU utilized across all NodeManager hosts.",
+ "default_section_name": "YARN_SUMMARY",
+ "widget_type": "GRAPH",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "cpu_system._sum",
+ "metric_path": "metrics/cpu/cpu_system._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "cpu_user._sum",
+ "metric_path": "metrics/cpu/cpu_user._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "cpu_nice._sum",
+ "metric_path": "metrics/cpu/cpu_nice._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "cpu_idle._sum",
+ "metric_path": "metrics/cpu/cpu_idle._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "cpu_wio._sum",
+ "metric_path": "metrics/cpu/cpu_wio._sum",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "CPU utilization",
+ "value": "${((cpu_system._sum + cpu_user._sum + cpu_nice._sum)/(cpu_system._sum + cpu_user._sum + cpu_nice._sum + cpu_idle._sum + cpu_wio._sum)) * 100}"
+ }
+ ],
+ "properties": {
+ "graph_type": "LINE",
+ "time_range": "1",
+ "display_unit": "%"
+ }
+ }
+ ]
+ },
+ {
+ "layout_name": "default_yarn_heatmap",
+ "display_name": "YARN Heatmaps",
+ "section_name": "YARN_HEATMAPS",
+ "widgetLayoutInfo": [
+ {
+ "widget_name": "YARN local disk space utilization per NodeManager",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc",
+ "metric_path": "metrics/yarn/GoodLocalDirsDiskUtilizationPerc",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc",
+ "metric_path": "metrics/yarn/GoodLogDirsDiskUtilizationPerc",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "YARN local disk space utilization per NodeManager",
+ "value": "${(yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc + yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc)/2}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Total Allocatable RAM Utilized per NodeManager",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.AllocatedGB",
+ "metric_path": "metrics/yarn/AllocatedGB",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.AvailableGB",
+ "metric_path": "metrics/yarn/AvailableGB",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Total Allocatable RAM Utilized per NodeManager",
+ "value": "${(yarn.NodeManagerMetrics.AllocatedGB/(yarn.NodeManagerMetrics.AvailableGB + yarn.NodeManagerMetrics.AllocatedGB)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Total Allocatable CPU Utilized per NodeManager",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.AllocatedVCores",
+ "metric_path": "metrics/yarn/AllocatedVCores",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.AvailableVCores",
+ "metric_path": "metrics/yarn/AvailableVCores",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Total Allocatable CPU Utilized per NodeManager",
+ "value": "${(yarn.NodeManagerMetrics.AllocatedVCores/(yarn.NodeManagerMetrics.AllocatedVCores + yarn.NodeManagerMetrics.AvailableVCores)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Container Failures",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersFailed._rate",
+ "metric_path": "metrics/yarn/ContainersFailed._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersCompleted._rate",
+ "metric_path": "metrics/yarn/ContainersCompleted._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersLaunched._rate",
+ "metric_path": "metrics/yarn/ContainersLaunched._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersIniting",
+ "metric_path": "metrics/yarn/ContainersIniting",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersKilled._rate",
+ "metric_path": "metrics/yarn/ContainersKilled._rate",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.ContainersRunning",
+ "metric_path": "metrics/yarn/ContainersRunning",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Container Failures",
+ "value": "${(yarn.NodeManagerMetrics.ContainersFailed._rate/(yarn.NodeManagerMetrics.ContainersFailed._rate + yarn.NodeManagerMetrics.ContainersCompleted._rate + yarn.NodeManagerMetrics.ContainersLaunched._rate + yarn.NodeManagerMetrics.ContainersIniting + yarn.NodeManagerMetrics.ContainersKilled._rate + yarn.NodeManagerMetrics.ContainersRunning)) * 100}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "NodeManager GC Time",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "Hadoop:service=NodeManager,name=JvmMetrics.GcTimeMillis",
+ "metric_path": "metrics/jvm/gcTimeMillis",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "NodeManager Garbage Collection Time",
+ "value": "${Hadoop:service=NodeManager,name=JvmMetrics.GcTimeMillis}"
+ }
+ ],
+ "properties": {
+ "display_unit": "ms",
+ "max_limit": "10000"
+ }
+ },
+ {
+ "widget_name": "NodeManager JVM Heap Memory Used",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "Hadoop:service=NodeManager,name=JvmMetrics.MemHeapUsedM",
+ "metric_path": "metrics/jvm/memHeapUsedM",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "NodeManager JVM Heap Memory Used",
+ "value": "${Hadoop:service=NodeManager,name=JvmMetrics.MemHeapUsedM}"
+ }
+ ],
+ "properties": {
+ "display_unit": "MB",
+ "max_limit": "512"
+ }
+ },
+ {
+ "widget_name": "Allocated Containers",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.AllocatedContainers",
+ "metric_path": "metrics/yarn/AllocatedContainers",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Allocated Containers",
+ "value": "${yarn.NodeManagerMetrics.AllocatedContainers}"
+ }
+ ],
+ "properties": {
+ "display_unit": "",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "NodeManager RAM Utilized",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.AllocatedGB",
+ "metric_path": "metrics/yarn/AllocatedGB",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "NodeManager RAM Utilized",
+ "value": "${yarn.NodeManagerMetrics.AllocatedGB}"
+ }
+ ],
+ "properties": {
+ "display_unit": "",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "NodeManager CPU Utilized",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.AllocatedVCores",
+ "metric_path": "metrics/yarn/AllocatedVCores",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "NodeManager CPU Utilized",
+ "value": "${yarn.NodeManagerMetrics.AllocatedVCores}"
+ }
+ ],
+ "properties": {
+ "display_unit": "",
+ "max_limit": "100"
+ }
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml
new file mode 100644
index 0000000..a70fad3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-env.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <!-- mapred-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>mapred-env template</display-name>
+ <description>This is the jinja template for mapred-env.sh file</description>
+ <value>
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}
+
+export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
+#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
+#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
+#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
+#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
+#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
+export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS"
+export HADOOP_OPTS="-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS"
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml
new file mode 100644
index 0000000..cef2b14
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration-mapred/mapred-site.xml
@@ -0,0 +1,134 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ <description>
+ Additional execution environment entries for map and reduce task processes.
+ This is not an additive property. You must preserve the original value if
+ you want your map and reduce tasks to have access to native libraries (compression, etc)
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.application.classpath</name>
+ <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <description>
+ CLASSPATH for MR applications. A comma-separated list of CLASSPATH
+ entries.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.application.framework.path</name>
+ <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value>-Dhdp.version=${hdp.version}</value>
+ <description>
+ Java opts for the MR App Master processes.
+ The following symbol, if present, will be interpolated: @taskid@ is replaced
+ by current TaskID. Any other occurrences of '@' will go unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+ -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+
+ Usage of -Djava.library.path can cause programs to no longer function if
+ hadoop native libraries are used. These values should instead be set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.admin.map.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.admin.reduce.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
+ <value>1</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
+ <value>1000</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
+ <value>30000</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.job.emit-timeline-data</name>
+ <value>false</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.bind-host</name>
+ <value>0.0.0.0</value>
+ <description/>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>mapreduce.jobhistory.recovery.enable</name>
+ <value>true</value>
+ <description>Enable the history server to store server state and recover
+ server state upon startup. If enabled then
+ mapreduce.jobhistory.recovery.store.class must be specified.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.recovery.store.class</name>
+ <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
+ <description>The HistoryServerStateStoreService class to store history server
+ state for recovery.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
+ <value>/hadoop/mapreduce/jhs</value>
+ <description>The URI where history server state will be stored if HistoryServerLeveldbSystemStateStoreService
+ is configured as the recovery storage class.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/capacity-scheduler.xml
new file mode 100644
index 0000000..4768e46
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/capacity-scheduler.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <!-- These configs were inherited from HDP 2.2 -->
+ <property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <description>
+ The ResourceCalculator implementation to be used to compare Resources in the scheduler.
+ The default i.e. org.apache.hadoop.yarn.util.resource.DefaultResourseCalculator only uses
+ Memory while DominantResourceCalculator uses Dominant-resource to compare multi-dimensional
+ resources such as Memory, CPU etc. A Java ResourceCalculator class name is expected.
+ </description>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <display-name>CPU Scheduling</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>org.apache.hadoop.yarn.util.resource.DominantResourceCalculator</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
+ <value>*</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- In HDP 2.3, yarn.scheduler.capacity.root.default-node-label-expression was deleted -->
+
+ <!-- These configs were inherited from HDP 2.5 -->
+ <property>
+ <name>capacity-scheduler</name>
+ <description>Enter key=value (one per line) for all properties of capacity-scheduler.xml</description>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-audit.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-audit.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-audit.xml
new file mode 100644
index 0000000..a6b1baa
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-audit.xml
@@ -0,0 +1,177 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ <description>Is Audit enabled?</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db</name>
+ <value>false</value>
+ <display-name>Audit to DB</display-name>
+ <description>Is Audit to DB enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.db</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.url</name>
+ <value>{{audit_jdbc_url}}</value>
+ <description>Audit DB JDBC URL</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.user</name>
+ <value>{{xa_audit_db_user}}</value>
+ <description>Audit DB JDBC User</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.password</name>
+ <value>crypted</value>
+ <property-type>PASSWORD</property-type>
+ <description>Audit DB JDBC Password</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.jdbc.driver</name>
+ <value>{{jdbc_driver}}</value>
+ <description>Audit DB JDBC Driver</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.credential.provider.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>Credential file store</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.db.batch.filespool.dir</name>
+ <value>/var/log/hadoop/yarn/audit/db/spool</value>
+ <description>/var/log/hadoop/yarn/audit/db/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <value>true</value>
+ <display-name>Audit to HDFS</display-name>
+ <description>Is Audit to HDFS enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <value>hdfs://NAMENODE_HOSTNAME:8020/ranger/audit</value>
+ <description>HDFS folder to write audit to, make sure the service user has requried permissions</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
+ <value>/var/log/hadoop/yarn/audit/hdfs/spool</value>
+ <description>/var/log/hadoop/yarn/audit/hdfs/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr</name>
+ <value>false</value>
+ <display-name>Audit to SOLR</display-name>
+ <description>Is Solr audit enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.solr</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.urls</name>
+ <value/>
+ <description>Solr URL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.urls</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.zookeepers</name>
+ <value>NONE</value>
+ <description>Solr Zookeeper string</description>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.zookeepers</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.batch.filespool.dir</name>
+ <value>/var/log/hadoop/yarn/audit/solr/spool</value>
+ <description>/var/log/hadoop/yarn/audit/solr/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.provider.summary.enabled</name>
+ <value>false</value>
+ <display-name>Audit provider summary enabled</display-name>
+ <description>Enable Summary audit?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-plugin-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-plugin-properties.xml
new file mode 100644
index 0000000..97867cc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-plugin-properties.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>policy_user</name>
+ <value>ambari-qa</value>
+ <display-name>Policy user for YARN</display-name>
+ <description>This user must be system user and also present at Ranger admin portal</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value/>
+ <description>Used for repository creation on ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>common.name.for.certificate</name>
+ <value/>
+ <description>Common name for certificate, this value should match what is specified in repo within ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger-yarn-plugin-enabled</name>
+ <value>No</value>
+ <display-name>Enable Ranger for YARN</display-name>
+ <description>Enable ranger yarn plugin ?</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>ranger-yarn-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_USERNAME</name>
+ <value>yarn</value>
+ <display-name>Ranger repository config user</display-name>
+ <description>Used for repository creation on ranger admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_PASSWORD</name>
+ <value>yarn</value>
+ <display-name>Ranger repository config password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Used for repository creation on ranger admin</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-policymgr-ssl.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-policymgr-ssl.xml
new file mode 100644
index 0000000..5410104
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-policymgr-ssl.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>/usr/hdp/current/hadoop-client/conf/ranger-yarn-plugin-keystore.jks</value>
+ <description>Java Keystore files</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.password</name>
+ <value>myKeyFilePassword</value>
+ <property-type>PASSWORD</property-type>
+ <description>password for keystore</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>/usr/hdp/current/hadoop-client/conf/ranger-yarn-plugin-truststore.jks</value>
+ <description>java truststore file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.password</name>
+ <value>changeit</value>
+ <property-type>PASSWORD</property-type>
+ <description>java truststore password</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java keystore credential file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>java truststore credential file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-security.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-security.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-security.xml
new file mode 100644
index 0000000..5f69962
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/ranger-yarn-security.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>ranger.plugin.yarn.service.name</name>
+ <value>{{repo_name}}</value>
+ <description>Name of the Ranger service containing policies for this Yarn instance</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.yarn.policy.source.impl</name>
+ <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+ <description>Class to retrieve policies from the source</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.yarn.policy.rest.url</name>
+ <value>{{policymgr_mgr_url}}</value>
+ <description>URL to Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.yarn.policy.rest.ssl.config.file</name>
+ <value>/etc/hadoop/conf/ranger-policymgr-ssl-yarn.xml</value>
+ <description>Path to the file containing SSL details to contact Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.yarn.policy.pollIntervalMs</name>
+ <value>30000</value>
+ <description>How often to poll for changes in policies?</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.yarn.policy.cache.dir</name>
+ <value>/etc/ranger/{{repo_name}}/policycache</value>
+ <description>Directory where Ranger policies are cached after successful retrieval from the source</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-env.xml
new file mode 100644
index 0000000..bbc2930
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-env.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true" supports_adding_forbidden="true">
+ <!-- These properties were inherited from HDP 2.1 -->
+ <property>
+ <name>apptimelineserver_heapsize</name>
+ <value>1024</value>
+ <display-name>AppTimelineServer Java heap size</display-name>
+ <description>Max heapsize for AppTimelineServer using a numerical value in the scale of MB</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <unit>MB</unit>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These properties were inherited from HDP 2.2 -->
+ <property>
+ <name>yarn_cgroups_enabled</name>
+ <value>false</value>
+ <description>You can use CGroups to isolate CPU-heavy processes in a Hadoop cluster.</description>
+ <display-name>CPU Isolation</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- These properties were inherited from HDP 2.3 -->
+ <property>
+ <name>is_supported_yarn_ranger</name>
+ <value>true</value>
+ <description>Set to false by default, needs to be set to true in stacks that use Ranger Yarn Plugin</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <!-- yarn-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>yarn-env template</display-name>
+ <description>This is the jinja template for yarn-env.sh file</description>
+ <value>
+ export HADOOP_YARN_HOME={{hadoop_yarn_home}}
+ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
+ export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
+ export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
+ export JAVA_HOME={{java64_home}}
+ export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
+
+ # We need to add the EWMA appender for the yarn daemons only;
+ # however, YARN_ROOT_LOGGER is shared by the yarn client and the
+ # daemons. This is restrict the EWMA appender to daemons only.
+ INVOKER="${0##*/}"
+ if [ "$INVOKER" == "yarn-daemon.sh" ]; then
+ export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA}
+ fi
+
+ # User for YARN daemons
+ export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+ # resolve links - $0 may be a softlink
+ export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+ # some Java parameters
+ # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+ if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+ fi
+
+ if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+ fi
+
+ JAVA=$JAVA_HOME/bin/java
+ JAVA_HEAP_MAX=-Xmx1000m
+
+ # For setting YARN specific HEAP sizes please use this
+ # Parameter and set appropriately
+ YARN_HEAPSIZE={{yarn_heapsize}}
+
+ # check envvars which might override default args
+ if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+ fi
+
+ # Resource Manager specific parameters
+
+ # Specify the max Heapsize for the ResourceManager using a numerical value
+ # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+ # the value to 1000.
+ # This value will be overridden by an Xmx setting specified in either YARN_OPTS
+ # and/or YARN_RESOURCEMANAGER_OPTS.
+ # If not specified, the default value will be picked from either YARN_HEAPMAX
+ # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+ export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
+
+ # Specify the JVM options to be used when starting the ResourceManager.
+ # These options will be appended to the options specified as YARN_OPTS
+ # and therefore may override any similar flags set in YARN_OPTS
+ #export YARN_RESOURCEMANAGER_OPTS=
+
+ # Node Manager specific parameters
+
+ # Specify the max Heapsize for the NodeManager using a numerical value
+ # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+ # the value to 1000.
+ # This value will be overridden by an Xmx setting specified in either YARN_OPTS
+ # and/or YARN_NODEMANAGER_OPTS.
+ # If not specified, the default value will be picked from either YARN_HEAPMAX
+ # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+ export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
+
+ # Specify the max Heapsize for the timeline server using a numerical value
+ # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+ # the value to 1024.
+ # This value will be overridden by an Xmx setting specified in either YARN_OPTS
+ # and/or YARN_TIMELINESERVER_OPTS.
+ # If not specified, the default value will be picked from either YARN_HEAPMAX
+ # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+ export YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}}
+
+ # Specify the JVM options to be used when starting the NodeManager.
+ # These options will be appended to the options specified as YARN_OPTS
+ # and therefore may override any similar flags set in YARN_OPTS
+ #export YARN_NODEMANAGER_OPTS=
+
+ # so that filenames w/ spaces are handled correctly in loops below
+ IFS=
+
+
+ # default log directory and file
+ if [ "$YARN_LOG_DIR" = "" ]; then
+ YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+ fi
+ if [ "$YARN_LOGFILE" = "" ]; then
+ YARN_LOGFILE='yarn.log'
+ fi
+
+ # default policy file for service-level authorization
+ if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+ fi
+
+ # restore ordinary behaviour
+ unset IFS
+
+
+ YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+ YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+ YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+ YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+ YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+ YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+ YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+ YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+ export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT"
+ export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT"
+ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+ fi
+ YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a62119ab/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-log4j.xml
new file mode 100644
index 0000000..9ac34f3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-log4j.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>yarn-log4j template</display-name>
+ <description>Custom log4j.properties</description>
+ <value>
+#Relative to Yarn Log Dir Prefix
+yarn.log.dir=.
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
+# Set the ResourceManager summary log filename
+yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log
+# Set the ResourceManager summary log level and appender
+yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# To enable AppSummaryLogging for the RM,
+# set yarn.server.resourcemanager.appsummary.logger to
+# LEVEL,RMSUMMARY in hadoop-env.sh
+
+# Appender for ResourceManager Application Summary Log
+# Requires the following properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
+log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
+log4j.appender.RMSUMMARY.MaxFileSize=256MB
+log4j.appender.RMSUMMARY.MaxBackupIndex=20
+log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+
+# Appender for viewing information for errors and warnings
+yarn.ewma.cleanupInterval=300
+yarn.ewma.messageAgeLimitSeconds=86400
+yarn.ewma.maxUniqueMessages=250
+log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
+log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
+log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
+log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
+
+# Audit logging for ResourceManager
+rm.audit.logger=${hadoop.root.logger}
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false
+log4j.appender.RMAUDIT=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.RMAUDIT.File=${yarn.log.dir}/rm-audit.log
+log4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.RMAUDIT.DatePattern=.yyyy-MM-dd
+
+# Audit logging for NodeManager
+nm.audit.logger=${hadoop.root.logger}
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger}
+log4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false
+log4j.appender.NMAUDIT=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.NMAUDIT.File=${yarn.log.dir}/nm-audit.log
+log4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout
+log4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.NMAUDIT.DatePattern=.yyyy-MM-dd
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
[44/50] [abbrv] ambari git commit: Revert "AMBARI-18966. Add check to
ensure we do not have @Transactional annotations on private methods. (Attila
Doroszlai via stoader)"
Posted by nc...@apache.org.
Revert "AMBARI-18966. Add check to ensure we do not have @Transactional annotations on private methods. (Attila Doroszlai via stoader)"
This reverts commit 9170b49933106a0af651530d827af4f4e73f1541.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6806edcd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6806edcd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6806edcd
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 6806edcd94dd5568b8a21ec757fa7fa668eec71c
Parents: 9170b49
Author: Toader, Sebastian <st...@hortonworks.com>
Authored: Wed Dec 7 10:44:23 2016 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Wed Dec 7 10:44:23 2016 +0100
----------------------------------------------------------------------
ambari-server/checkstyle.xml | 17 -----------------
ambari-server/pom.xml | 4 ----
.../internal/AlertTargetResourceProvider.java | 2 +-
.../ClusterStackVersionResourceProvider.java | 4 ++--
.../internal/UpgradeResourceProvider.java | 2 +-
.../apache/ambari/server/orm/dao/AlertsDAO.java | 6 +++---
.../apache/ambari/server/state/ServiceImpl.java | 2 +-
.../ambari/server/state/cluster/ClustersImpl.java | 2 +-
.../apache/ambari/server/state/host/HostImpl.java | 2 +-
.../state/services/RetryUpgradeActionService.java | 2 +-
.../svccomphost/ServiceComponentHostImpl.java | 2 +-
.../ambari/server/upgrade/UpgradeCatalog240.java | 4 ++--
12 files changed, 14 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/checkstyle.xml
----------------------------------------------------------------------
diff --git a/ambari-server/checkstyle.xml b/ambari-server/checkstyle.xml
deleted file mode 100644
index 81f6380..0000000
--- a/ambari-server/checkstyle.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0"?>
-<!-- Licensed under the Apache License, Version 2.0 (the "License"); you
- may not use this file except in compliance with the License. You may obtain
- a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
- required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License.
- See accompanying LICENSE file. -->
-<!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
- "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
-<module name="Checker">
- <module name="TreeWalker">
- <module name="AvoidTransactionalOnPrivateMethodsCheck"/>
- </module>
-</module>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index e8de84e..69ab9d0 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -692,10 +692,6 @@
<artifactId>jetty-maven-plugin</artifactId>
<version>${jetty.version}</version>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
</plugins>
<resources>
<resource>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
index 4010528..8e5674e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AlertTargetResourceProvider.java
@@ -374,7 +374,7 @@ public class AlertTargetResourceProvider extends
*/
@Transactional
@SuppressWarnings("unchecked")
- void updateAlertTargets(long alertTargetId,
+ private void updateAlertTargets(long alertTargetId,
Map<String, Object> requestMap)
throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index d417ec2..42aae84 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -442,7 +442,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
+ private void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId,
String desiredRepoVersion, RepositoryVersionState repoState)
throws AmbariException, SystemException {
final String clusterName = cluster.getClusterName();
@@ -482,7 +482,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
@Transactional
- RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
+ private RequestStageContainer createOrchestration(Cluster cluster, StackId stackId,
List<Host> hosts, RepositoryVersionEntity repoVersionEnt, Map<String, Object> propertyMap)
throws AmbariException, SystemException {
final AmbariManagementController managementController = getManagementController();
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 67be152..5931743 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -972,7 +972,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
@Transactional
- UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
+ private UpgradeEntity createUpgradeInsideTransaction(Cluster cluster,
RequestStageContainer request,
UpgradeEntity upgradeEntity) throws AmbariException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
index e8c4b5f..d187947 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java
@@ -1492,7 +1492,7 @@ public class AlertsDAO implements Cleanable {
* @return a long representing the number of affected (deleted) records
*/
@Transactional
- int cleanAlertNoticesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
+ private int cleanAlertNoticesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
LOG.info("Deleting AlertNotice entities before date " + new Date(beforeDateMillis));
EntityManager entityManager = m_entityManagerProvider.get();
List<Integer> ids = findAllAlertHistoryIdsBeforeDate(clusterId, beforeDateMillis);
@@ -1523,7 +1523,7 @@ public class AlertsDAO implements Cleanable {
* @return a long representing the number of affected (deleted) records
*/
@Transactional
- int cleanAlertCurrentsForClusterBeforeDate(long clusterId, long beforeDateMillis) {
+ private int cleanAlertCurrentsForClusterBeforeDate(long clusterId, long beforeDateMillis) {
LOG.info("Deleting AlertCurrent entities before date " + new Date(beforeDateMillis));
EntityManager entityManager = m_entityManagerProvider.get();
List<Integer> ids = findAllAlertHistoryIdsBeforeDate(clusterId, beforeDateMillis);
@@ -1553,7 +1553,7 @@ public class AlertsDAO implements Cleanable {
*/
@Transactional
- int cleanAlertHistoriesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
+ private int cleanAlertHistoriesForClusterBeforeDate(Long clusterId, long beforeDateMillis) {
return executeQuery("AlertHistoryEntity.removeInClusterBeforeDate", AlertHistoryEntity.class, clusterId, beforeDateMillis);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
index e223eed..f87b99c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
@@ -451,7 +451,7 @@ public class ServiceImpl implements Service {
}
@Transactional
- void persistEntities(ClusterServiceEntity serviceEntity) {
+ private void persistEntities(ClusterServiceEntity serviceEntity) {
long clusterId = cluster.getClusterId();
ClusterEntity clusterEntity = clusterDAO.findById(clusterId);
serviceEntity.setClusterEntity(clusterEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 1de10f9..8e5f37e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@ -183,7 +183,7 @@ public class ClustersImpl implements Clusters {
*/
@Inject
@Transactional
- void loadClustersAndHosts() {
+ private void loadClustersAndHosts() {
List<HostEntity> hostEntities = hostDAO.findAll();
for (HostEntity hostEntity : hostEntities) {
Host host = hostFactory.create(hostEntity);
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index a444f5d..dd5e635 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@ -954,7 +954,7 @@ public class HostImpl implements Host {
}
@Transactional
- void persistEntities(HostEntity hostEntity) {
+ private void persistEntities(HostEntity hostEntity) {
hostDAO.create(hostEntity);
if (!hostEntity.getClusterEntities().isEmpty()) {
for (ClusterEntity clusterEntity : hostEntity.getClusterEntities()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
index a92aa04..1ea5558 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/RetryUpgradeActionService.java
@@ -190,7 +190,7 @@ public class RetryUpgradeActionService extends AbstractScheduledService {
* @param requestId Request Id to search tasks for.
*/
@Transactional
- void retryHoldingCommandsInRequest(Long requestId) {
+ private void retryHoldingCommandsInRequest(Long requestId) {
if (requestId == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 5225598..c1655aa 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -1313,7 +1313,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
}
@Transactional
- void persistEntities(HostEntity hostEntity, HostComponentStateEntity stateEntity,
+ private void persistEntities(HostEntity hostEntity, HostComponentStateEntity stateEntity,
HostComponentDesiredStateEntity desiredStateEntity) {
ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName(
serviceComponent.getClusterId(), serviceComponent.getServiceName(),
http://git-wip-us.apache.org/repos/asf/ambari/blob/6806edcd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index b7e248a..c113fff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -1563,7 +1563,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
* @throws SQLException
*/
@Transactional
- void updateServiceComponentDesiredStateTableDDL() throws SQLException {
+ private void updateServiceComponentDesiredStateTableDDL() throws SQLException {
if (dbAccessor.tableHasPrimaryKey(SERVICE_COMPONENT_DS_TABLE, ID)) {
LOG.info("Skipping {} table Primary Key modifications since the new {} column already exists",
SERVICE_COMPONENT_DS_TABLE, ID);
@@ -2785,7 +2785,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
* instead of cluster_name
*/
@Transactional
- void updateViewInstanceTable() throws SQLException {
+ private void updateViewInstanceTable() throws SQLException {
try {
if (Long.class.equals(dbAccessor.getColumnClass(VIEWINSTANCE_TABLE, CLUSTER_HANDLE_COLUMN))) {
LOG.info(String.format("%s column is already numeric. Skipping an update of %s table.", CLUSTER_HANDLE_COLUMN, VIEWINSTANCE_TABLE));
[26/50] [abbrv] ambari git commit: AMBARI-19103. Support for multi
lines tabs (onechiporenko)
Posted by nc...@apache.org.
AMBARI-19103. Support for multi lines tabs (onechiporenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14c39691
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14c39691
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14c39691
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 14c39691e06bde1d158c75e65251dc1dc1ba20e9
Parents: 353b2ac
Author: Oleg Nechiporenko <on...@apache.org>
Authored: Tue Dec 6 12:14:55 2016 +0200
Committer: Oleg Nechiporenko <on...@apache.org>
Committed: Tue Dec 6 12:14:55 2016 +0200
----------------------------------------------------------------------
ambari-web/app/styles/theme/bootstrap-ambari.css | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/14c39691/ambari-web/app/styles/theme/bootstrap-ambari.css
----------------------------------------------------------------------
diff --git a/ambari-web/app/styles/theme/bootstrap-ambari.css b/ambari-web/app/styles/theme/bootstrap-ambari.css
index d10a13b..738fa59 100644
--- a/ambari-web/app/styles/theme/bootstrap-ambari.css
+++ b/ambari-web/app/styles/theme/bootstrap-ambari.css
@@ -452,7 +452,9 @@ h2.table-title {
.nav.nav-tabs li a:active,
.nav.nav-tabs li a:focus {
color: #333;
- border-width: 0;
+ border-top-width: 0;
+ border-left-width: 0;
+ border-right-width: 0;
background: none;
}
.nav.nav-tabs li a .badge.badge-important {