You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2016/03/11 06:47:17 UTC
[09/21] ambari git commit: AMBARI-15228. Ambari overwrites
permissions on HDFS directories (aonishuk)
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
index 18205f7..944c540 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
@@ -27,6 +27,7 @@ class TestHawqMaster(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = 'HAWQ/2.0.0/package'
STACK_VERSION = '2.3'
GPADMIN = 'gpadmin'
+ DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
def __asserts_for_configure(self):
@@ -167,6 +168,7 @@ class TestHawqMaster(RMFTestCase):
)
self.assertResourceCalled('HdfsResource', '/hawq_default',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
default_fs = u'hdfs://c6401.ambari.apache.org:8020',
hdfs_site = self.getConfig()['configurations']['hdfs-site'],
type = 'directory',
@@ -183,6 +185,7 @@ class TestHawqMaster(RMFTestCase):
)
self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
default_fs = u'hdfs://c6401.ambari.apache.org:8020',
hdfs_site = self.getConfig()['configurations']['hdfs-site'],
action = ['execute'],
@@ -230,6 +233,7 @@ class TestHawqMaster(RMFTestCase):
)
self.assertResourceCalled('HdfsResource', '/hawq_default',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
default_fs = u'hdfs://c6401.ambari.apache.org:8020',
hdfs_site = self.getConfig()['configurations']['hdfs-site'],
type = 'directory',
@@ -246,6 +250,7 @@ class TestHawqMaster(RMFTestCase):
)
self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
default_fs = u'hdfs://c6401.ambari.apache.org:8020',
hdfs_site = self.getConfig()['configurations']['hdfs-site'],
action = ['execute'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
index 78ae3fe..7a3f010 100644
--- a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
@@ -26,6 +26,7 @@ from only_for_platform import not_for_platform, PLATFORM_WINDOWS
class TestMahoutClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "MAHOUT/1.0.0.2.3/package"
STACK_VERSION = "2.3"
+ DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
@@ -41,6 +42,7 @@ class TestMahoutClient(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeoutput',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
@@ -52,6 +54,7 @@ class TestMahoutClient(RMFTestCase):
type = 'directory',
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
@@ -64,6 +67,7 @@ class TestMahoutClient(RMFTestCase):
action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
)
self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput/sample-mahout-test.txt',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
@@ -77,6 +81,7 @@ class TestMahoutClient(RMFTestCase):
action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
)
self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
index c23fd96..674f30d 100644
--- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
@@ -28,6 +28,7 @@ from only_for_platform import not_for_platform, PLATFORM_WINDOWS
class TestSparkThriftServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
STACK_VERSION = "2.3"
+ DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
def test_configure_default(self, copy_to_hdfs_mock):
@@ -91,6 +92,7 @@ class TestSparkThriftServer(RMFTestCase):
mode = 0775
)
self.assertResourceCalled('HdfsResource', '/user/spark',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
@@ -107,6 +109,7 @@ class TestSparkThriftServer(RMFTestCase):
mode = 0775,
)
self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
index 9197cf1..a274d45 100644
--- a/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
+++ b/ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py
@@ -36,6 +36,7 @@ origin_exists = os.path.exists
class TestAts(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
STACK_VERSION = "2.3"
+ DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/application_timeline_server.py",
@@ -151,6 +152,7 @@ class TestAts(RMFTestCase):
cd_access = 'a',
)
self.assertResourceCalled('HdfsResource', '/ats',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/bin',
keytab = UnknownConfigurationMock(),
@@ -169,6 +171,7 @@ class TestAts(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('HdfsResource', '/ats/done',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/bin',
keytab = UnknownConfigurationMock(),
@@ -186,6 +189,7 @@ class TestAts(RMFTestCase):
mode = 0700,
)
self.assertResourceCalled('HdfsResource', '/ats',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/bin',
keytab = UnknownConfigurationMock(),
@@ -204,6 +208,7 @@ class TestAts(RMFTestCase):
mode = 0755,
)
self.assertResourceCalled('HdfsResource', '/ats/active',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/bin',
keytab = UnknownConfigurationMock(),
@@ -221,6 +226,7 @@ class TestAts(RMFTestCase):
mode = 01777,
)
self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
security_enabled = False,
hadoop_bin_dir = '/usr/bin',
keytab = UnknownConfigurationMock(),
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
index 2ae9584..3fe1235 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/ats_1_5.json
@@ -3,7 +3,8 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
- "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
+ "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
"ambari_db_rca_password": "mapred",
"ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
"repo_info": "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
@@ -531,6 +532,7 @@
"xml_configurations_supported" : "false"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"hdfs_user_principal" : "",
"hdfs_user_keytab" : "",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/default.hbasedecom.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.hbasedecom.json b/ambari-server/src/test/python/stacks/2.3/configs/default.hbasedecom.json
index db157af..fff0024 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.hbasedecom.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.hbasedecom.json
@@ -3,6 +3,7 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"agent_stack_retry_count": "5",
"agent_stack_retry_on_unavailability": "false",
"jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
@@ -428,6 +429,7 @@
"min_user_id": "1000"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"ignore_groupsusers_create": "false",
"smokeuser": "ambari-qa",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index 253e833..d5102dc 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -3,6 +3,7 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"agent_stack_retry_count": "5",
"agent_stack_retry_on_unavailability": "false",
"jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
@@ -63,6 +64,7 @@
"yarn.resourcemanager.scheduler.address": "c6401.ambari.apache.org:8030"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"ignore_groupsusers_create": "false",
"smokeuser": "ambari-qa",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json b/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
index 241547d..99d43cb 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
@@ -82,6 +82,7 @@
"clusterName": "phd",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
"ambari_db_rca_password": "mapred",
"java_home": "/usr/jdk64/jdk1.8.0_60",
@@ -1027,6 +1028,7 @@
"mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
"ignore_groupsusers_create": "false",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/hbase_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/hbase_default.json b/ambari-server/src/test/python/stacks/2.3/configs/hbase_default.json
index f560704..0da58ce 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/hbase_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/hbase_default.json
@@ -39,6 +39,7 @@
"clusterName": "c1",
"hostname": "c6405.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"agent_stack_retry_count": "5",
"agent_stack_retry_on_unavailability": "false",
"jdk_location": "http://c6405.ambari.apache.org:8080/resources/",
@@ -313,6 +314,7 @@
"content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4
j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN
GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"ignore_groupsusers_create": "false",
"kerberos_domain": "EXAMPLE.COM",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/hbase_secure.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/hbase_secure.json b/ambari-server/src/test/python/stacks/2.3/configs/hbase_secure.json
index 3a8133a..8ecb91b 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/hbase_secure.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/hbase_secure.json
@@ -44,6 +44,7 @@
"clusterName": "c1",
"hostname": "c6405.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"agent_stack_retry_count": "5",
"agent_stack_retry_on_unavailability": "false",
"jdk_location": "http://c6405.ambari.apache.org:8080/resources/",
@@ -578,6 +579,7 @@
"content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nversion: 1\n\n# Please replace with site specific values\ndn: dc=hadoop,dc=apache,dc=org\nobjectclass: organization\nobjectclass: dcObject\no: Hadoop\ndc: hadoop\n\n#
Entry for a sample people container\n# Please replace with site specific values\ndn: ou=people,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass:organizationalUnit\nou: people\n\n# Entry for a sample end user\n# Please replace with site specific values\ndn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass:person\nobjectclass:organizationalPerson\nobjectclass:inetOrgPerson\ncn: Guest\nsn: User\nuid: guest\nuserPassword:guest-password\n\n# entry for sample user admin\ndn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass:person\nobjectclass:organizationalPerson\nobjectclass:inetOrgPerson\ncn: Admin\nsn: Admin\nuid: admin\nuserPassword:admin-password\n\n# entry for sample user sam\ndn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass:person\nobjectclass:organizationalPerson\nobjectclass:inetOrgPerson\ncn: sam\nsn: sam\nuid: sam\nuserPassword:sam-password\n\n# entry for sample user tom\ndn: uid=tom,
ou=people,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass:person\nobjectclass:organizationalPerson\nobjectclass:inetOrgPerson\ncn: tom\nsn: tom\nuid: tom\nuserPassword:tom-password\n\n# create FIRST Level groups branch\ndn: ou=groups,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass:organizationalUnit\nou: groups\ndescription: generic groups branch\n\n# create the analyst group under groups\ndn: cn=analyst,ou=groups,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass: groupofnames\ncn: analyst\ndescription:analyst group\nmember: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org\nmember: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org\n\n\n# create the scientist group under groups\ndn: cn=scientist,ou=groups,dc=hadoop,dc=apache,dc=org\nobjectclass:top\nobjectclass: groupofnames\ncn: scientist\ndescription: scientist group\nmember: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "true",
"ignore_groupsusers_create": "false",
"kerberos_domain": "EXAMPLE.COM",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json
index 1f6afe4..10cd9a4 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/pxf_default.json
@@ -3,6 +3,7 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"agent_stack_retry_count": "5",
"agent_stack_retry_on_unavailability": "false",
"jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
@@ -52,6 +53,7 @@
"content": " profile"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"ignore_groupsusers_create": "false",
"smokeuser": "ambari-qa",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json b/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
index 80411ec..ce982a1 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
@@ -3,6 +3,7 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
"jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
"ambari_db_rca_password": "mapred",
"ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
@@ -199,6 +200,7 @@
"yarn.resourcemanager.scheduler.address": "c6401.ambari.apache.org:8030"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"ignore_groupsusers_create": "false",
"smokeuser": "ambari-qa",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json b/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json
index 2de83a8..72afd78 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/storm_default.json
@@ -21,7 +21,8 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
- "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
+ "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
"ambari_db_rca_password": "mapred",
"java_home": "/usr/jdk64/jdk1.8.0_40",
"ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
@@ -173,6 +174,7 @@
"content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4
j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN
GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "false",
"ignore_groupsusers_create": "false",
"kerberos_domain": "EXAMPLE.COM",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/ambari-server/src/test/python/stacks/2.3/configs/storm_default_secure.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/storm_default_secure.json b/ambari-server/src/test/python/stacks/2.3/configs/storm_default_secure.json
index 3b92500..c1b7e10 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/storm_default_secure.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/storm_default_secure.json
@@ -21,7 +21,8 @@
"clusterName": "c1",
"hostname": "c6401.ambari.apache.org",
"hostLevelParams": {
- "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
+ "not_managed_hdfs_path_list": "[\"/apps/hive/warehouse\",\"/apps/falcon\",\"/mr-history/done\",\"/app-logs\",\"/tmp\"]",
+ "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
"ambari_db_rca_password": "mapred",
"java_home": "/usr/jdk64/jdk1.8.0_40",
"ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
@@ -184,6 +185,7 @@
"content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE\n\n# Example with rolling log file\n#log4
j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File=zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize=10MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex=10\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLIN
GFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
},
"cluster-env": {
+ "managed_hdfs_resource_property_names": "",
"security_enabled": "true",
"ignore_groupsusers_create": "false",
"kerberos_domain": "EXAMPLE.COM",
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
index d774ab8..9ef7660 100644
--- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
+++ b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
@@ -44,6 +44,7 @@ public class Resource {
private boolean recursiveChown;
private boolean recursiveChmod;
private boolean changePermissionforParents;
+ private boolean manageIfExists;
public String getSource() {
return source;
@@ -125,16 +126,22 @@ public class Resource {
this.changePermissionforParents = changePermissionforParents;
}
-
-
-
+ public boolean isManageIfExists() {
+ return manageIfExists;
+ }
+
+ public void setManageIfExists(boolean manageIfExists) {
+ this.manageIfExists = manageIfExists;
+ }
+
@Override
public String toString() {
return "Resource [source=" + source + ", target=" + target + ", type="
+ type + ", action=" + action + ", owner=" + owner + ", group=" + group
+ ", mode=" + mode + ", recursiveChown=" + recursiveChown
+ ", recursiveChmod=" + recursiveChmod
- + ", changePermissionforParents=" + changePermissionforParents + "]";
+ + ", changePermissionforParents=" + changePermissionforParents
+ + ", manageIfExists=" + manageIfExists + "]";
}
/*
http://git-wip-us.apache.org/repos/asf/ambari/blob/66267961/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
index 0c10ca7..e4656c7 100644
--- a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
+++ b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
@@ -72,6 +72,12 @@ public class Runner {
Resource.checkResourceParameters(resource, dfs);
Path pathHadoop = new Path(resource.getTarget());
+ if (!resource.isManageIfExists() && dfs.exists(pathHadoop)) {
+ System.out.println("Skipping the operation for not managed DFS directory " + resource.getTarget() +
+ " since immutable_paths contains it.");
+ continue;
+ }
+
if (resource.getAction().equals("create")) {
// 5 - Create
Resource.createResource(resource, dfs, pathHadoop);