You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/06 21:59:05 UTC
ambari git commit: AMBARI-19328. Fix permissions for Druid config
files and data directories (Nishant Bangarwa via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 212ba9136 -> 4780973a5
AMBARI-19328. Fix permissions for Druid config files and data directories (Nishant Bangarwa via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4780973a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4780973a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4780973a
Branch: refs/heads/branch-2.5
Commit: 4780973a5627723a22e735b32c2c76cf009a9119
Parents: 212ba91
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Jan 6 13:25:21 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Jan 6 13:25:21 2017 -0800
----------------------------------------------------------------------
.../0.9.2/configuration/druid-historical.xml | 4 +-
.../0.9.2/configuration/druid-middlemanager.xml | 2 +-
.../DRUID/0.9.2/package/scripts/druid.py | 36 ++++++++--
.../DRUID/0.9.2/package/scripts/params.py | 2 +
.../test/python/stacks/2.6/DRUID/test_druid.py | 74 +++++++++++++-------
.../test/python/stacks/2.6/configs/default.json | 6 +-
6 files changed, 87 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
index 0545e4c..9b65404 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml
@@ -61,7 +61,7 @@
</property>
<property>
<name>druid.segmentCache.locations</name>
- <value>[{"path":"/apps/druid/segmentCache","maxSize"\:300000000000}]</value>
+ <value>[{"path":"/apps/druid/segmentCache","maxSize":300000000000}]</value>
<description>Segments assigned to a Historical node are first stored on the local file system (in a disk cache) and
then served by the Historical node. These locations define where that local cache resides.
</description>
@@ -78,7 +78,7 @@
</property>
<property>
<name>druid.segmentCache.infoDir</name>
- <value>/apps/druid/segmentCache</value>
+ <value>/apps/druid/segmentCache/info_dir</value>
<description>Historical nodes keep track of the segments they are serving so that when the process is restarted they
can reload the same segments without waiting for the Coordinator to reassign. This path defines where this
metadata is kept. Directory will be created if needed.
http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml
index 967438e..921d16d 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml
@@ -55,7 +55,7 @@
</property>
<property>
<name>druid.indexer.task.baseTaskDir</name>
- <value>/tmp/persistent/tasks</value>
+ <value>/apps/druid/tasks</value>
<description>
Base temporary working directory for druid tasks.
</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
index 6ddedac..4a4f1c5 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
@@ -16,13 +16,13 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import json
import os
from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.core.resources.system import Directory, Execute, File
from resource_management.core.source import DownloadSource
from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions import format
-from resource_management.libraries.resources import XmlConfig
from resource_management.libraries.functions.show_logs import show_logs
from resource_management.core.logger import Logger
@@ -34,7 +34,8 @@ def druid(upgrade_type=None, nodeType=None):
# Environment Variables
File(format("{params.druid_conf_dir}/druid-env.sh"),
owner=params.druid_user,
- content=InlineTemplate(params.druid_env_sh_template)
+ content=InlineTemplate(params.druid_env_sh_template),
+ mode = 0700
)
# common config
@@ -60,6 +61,7 @@ def druid(upgrade_type=None, nodeType=None):
properties=druid_common_config,
owner=params.druid_user,
group=params.user_group,
+ mode = 0600
)
Logger.info("Created common.runtime.properties")
@@ -92,6 +94,7 @@ def druid(upgrade_type=None, nodeType=None):
properties=node_config,
owner=params.druid_user,
group=params.user_group,
+ mode = 0600
)
Logger.info(format("Created druid-{node_type_lowercase} runtime.properties"))
@@ -177,17 +180,38 @@ def create_hadoop_directory(hadoop_dir):
def ensure_base_directories():
import params
Directory(
- [params.druid_log_dir, params.druid_pid_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir,
- params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir,
- params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir],
+ [params.druid_log_dir, params.druid_pid_dir],
mode=0755,
- cd_access='a',
owner=params.druid_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True,
)
+ Directory(
+ [params.druid_conf_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir,
+ params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir,
+ params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir,
+ params.druid_tasks_dir],
+ mode=0700,
+ owner=params.druid_user,
+ group=params.user_group,
+ create_parents=True,
+ recursive_ownership=True,
+ )
+
+ segment_cache_locations = json.loads(params.druid_segment_cache_locations)
+ for segment_cache_location in segment_cache_locations:
+ Directory(
+ segment_cache_location["path"],
+ mode=0700,
+ owner=params.druid_user,
+ group=params.user_group,
+ create_parents=True,
+ recursive_ownership=True,
+ )
+
+
def get_daemon_cmd(params=None, node_type=None, command=None):
return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}')
http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
index 8292634..c83b16e 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
@@ -67,6 +67,8 @@ druid_router_conf_dir = druid_conf_dir + "/router"
druid_extensions_dir = druid_home + "/extensions"
druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies"
druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir']
+druid_segment_cache_locations = config['configurations']['druid-historical']['druid.segmentCache.locations']
+druid_tasks_dir = config['configurations']['druid-middlemanager']['druid.indexer.task.baseTaskDir']
druid_user = config['configurations']['druid-env']['druid_user']
druid_log_dir = config['configurations']['druid-env']['druid_log_dir']
druid_classpath = config['configurations']['druid-env']['druid_classpath']
http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
index 95cbb94..104dce6 100644
--- a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
+++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
@@ -321,7 +321,6 @@ class TestDruid(RMFTestCase):
self.assertResourceCalled('Directory', '/var/log/druid',
mode=0755,
- cd_access='a',
owner='druid',
group='hadoop',
create_parents=True,
@@ -330,7 +329,14 @@ class TestDruid(RMFTestCase):
self.assertResourceCalled('Directory', '/var/run/druid',
mode=0755,
- cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf'),
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -338,8 +344,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'),
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -347,8 +352,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'),
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -356,8 +360,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'),
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -365,8 +368,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'),
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -374,8 +376,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'),
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -383,8 +384,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'),
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -392,8 +392,23 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'),
- mode=0755,
- cd_access='a',
+ mode=0700,
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', '/apps/druid/segmentCache/info_dir',
+ mode=0700,
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', '/apps/druid/tasks',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -401,8 +416,7 @@ class TestDruid(RMFTestCase):
)
self.assertResourceCalled('Directory', '/apps/druid/segmentCache',
- mode=0755,
- cd_access='a',
+ mode=0700,
owner='druid',
group='hadoop',
create_parents=True,
@@ -411,7 +425,8 @@ class TestDruid(RMFTestCase):
self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'),
owner = 'druid',
- content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content'])
+ content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']),
+ mode = 0700
)
druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common'])
druid_common_config['druid.host'] = 'c6401.ambari.apache.org'
@@ -424,7 +439,8 @@ class TestDruid(RMFTestCase):
dir=format("/usr/hdp/current/{role}/conf/_common"),
properties=druid_common_config,
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'),
@@ -445,7 +461,8 @@ class TestDruid(RMFTestCase):
dir=format('/usr/hdp/current/{role}/conf/coordinator'),
properties=self.getConfig()['configurations']['druid-coordinator'],
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"),
@@ -463,7 +480,8 @@ class TestDruid(RMFTestCase):
dir=format('/usr/hdp/current/{role}/conf/overlord'),
properties=self.getConfig()['configurations']['druid-overlord'],
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"),
@@ -481,7 +499,8 @@ class TestDruid(RMFTestCase):
dir=format('/usr/hdp/current/{role}/conf/historical'),
properties=self.getConfig()['configurations']['druid-historical'],
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"),
@@ -500,7 +519,8 @@ class TestDruid(RMFTestCase):
dir=format('/usr/hdp/current/{role}/conf/broker'),
properties=self.getConfig()['configurations']['druid-broker'],
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"),
@@ -519,7 +539,8 @@ class TestDruid(RMFTestCase):
dir=format('/usr/hdp/current/{role}/conf/middleManager'),
properties=self.getConfig()['configurations']['druid-middlemanager'],
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"),
@@ -537,7 +558,8 @@ class TestDruid(RMFTestCase):
dir=format('/usr/hdp/current/{role}/conf/router'),
properties=self.getConfig()['configurations']['druid-router'],
owner='druid',
- group='hadoop'
+ group='hadoop',
+ mode = 0600
)
self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/4780973a/ambari-server/src/test/python/stacks/2.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json
index c3f1d55..fa12757 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json
@@ -405,7 +405,8 @@
"druid.extensions.loadList": "[\"mysql-metadata-storage\", \"druid-datasketches\"]"
},
"druid-historical" : {
- "druid.segmentCache.infoDir" : "/apps/druid/segmentCache"
+ "druid.segmentCache.infoDir" : "/apps/druid/segmentCache/info_dir",
+ "druid.segmentCache.locations" :"[{\"path\":\"/apps/druid/segmentCache\",\"maxSize\":300000000000}]"
},
"druid-coordinator" : {
"druid.service" : "druid/coordinator"
@@ -418,7 +419,8 @@
},
"druid-middlemanager" : {
"druid.service" : "druid/middlemanager",
- "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing"
+ "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing",
+ "druid.indexer.task.baseTaskDir" : "/apps/druid/tasks"
},
"druid-router" : {
"druid.service" : "druid/router"