You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/05/17 22:27:23 UTC
[ambari] branch trunk updated: [AMBARI-23883] Log Feeder: ambari services should be able to generate … (#1309)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5e67052 [AMBARI-23883] Log Feeder: ambari services should be able to generate … (#1309)
5e67052 is described below
commit 5e6705292e55f80fea054b341ce6c76adfebc0bb
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Fri May 18 00:27:20 2018 +0200
[AMBARI-23883] Log Feeder: ambari services should be able to generate … (#1309)
* AMBARI-23883. Log Feeder: ambari services should be able to generate logfeeder configs outside of after-INSTALL hook.
* AMBARI-23883. Add docs.
* AMBARI-23883. Add logsearch + infra-solr input patterns and remove env from method params
* AMBARI-23883. Support pattern generation for logfeeder itself.
---
.../src/main/python/ambari_commons/constants.py | 2 +
.../functions/generate_logfeeder_input_config.py | 44 ++++++++++++++++++++++
.../0.1.0/package/scripts/infra_solr.py | 5 +++
.../0.5.0/package/scripts/setup_logfeeder.py | 2 +
.../0.5.0/package/scripts/setup_logsearch.py | 3 ++
.../stack-hooks/after-INSTALL/scripts/params.py | 3 +-
.../stacks/2.4/AMBARI_INFRA/test_infra_solr.py | 11 ++++++
.../python/stacks/2.4/LOGSEARCH/test_logfeeder.py | 10 +++++
.../python/stacks/2.4/LOGSEARCH/test_logsearch.py | 13 +++++++
9 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/ambari-common/src/main/python/ambari_commons/constants.py b/ambari-common/src/main/python/ambari_commons/constants.py
index f1ab4a2..abb07da 100644
--- a/ambari-common/src/main/python/ambari_commons/constants.py
+++ b/ambari-common/src/main/python/ambari_commons/constants.py
@@ -26,6 +26,8 @@ UPGRADE_TYPE_HOST_ORDERED = "host_ordered"
AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"
+LOGFEEDER_CONF_DIR = "/usr/lib/ambari-logsearch-logfeeder/conf"
+
class SERVICE:
"""
Constants for service names to avoid hardcoding strings.
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/generate_logfeeder_input_config.py b/ambari-common/src/main/python/resource_management/libraries/functions/generate_logfeeder_input_config.py
new file mode 100755
index 0000000..84ab492
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/generate_logfeeder_input_config.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+from resource_management.core.logger import Logger
+from ambari_commons.constants import LOGFEEDER_CONF_DIR
+from resource_management.core.resources import File, Directory
+
+__all__ = ["generate_logfeeder_input_config"]
+
+def generate_logfeeder_input_config(type, content):
+ """
+ :param type: type of the logfeeder input config (most likely a service name: hdfs),
+ it will be generated as input.config-<type>.json in logfeeder config folder
+ :param content: generated template for the input config json file (you can use Template or InlineTemplate)
+ """
+ import params
+ Directory(LOGFEEDER_CONF_DIR,
+ mode=0755,
+ cd_access='a',
+ create_parents=True
+ )
+ input_file_name = 'input.config-' + type + '.json'
+ Logger.info("Generate Log Feeder config file: " + os.path.join(LOGFEEDER_CONF_DIR, input_file_name))
+ File(os.path.join(LOGFEEDER_CONF_DIR, input_file_name),
+ content=content,
+ mode=0644
+ )
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
index 363f8f4..a11f207 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
@@ -19,13 +19,16 @@ limitations under the License.
import sys
from resource_management.core.logger import Logger
+from resource_management.core.source import Template
from resource_management.core.resources.system import Execute, File
from resource_management.core.resources.zkmigrator import ZkMigrator
from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.get_user_call_output import get_user_call_output
from resource_management.libraries.functions.show_logs import show_logs
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config
from collection import backup_collection, restore_collection
from migrate import migrate_index
@@ -48,6 +51,8 @@ class InfraSolr(Script):
env.set_params(params)
self.configure(env)
+ generate_logfeeder_input_config('ambari-infra', Template("input.config-ambari-infra.json.j2", extra_imports=[default]))
+
setup_solr_znode_env()
start_cmd = format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} -Dsolr.kerberos.name.rules=\'{infra_solr_kerberos_name_rules}\' >> {infra_solr_log} 2>&1') \
if params.security_enabled else format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} >> {infra_solr_log} 2>&1')
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index 52b1d76..76ec5c5 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -21,6 +21,7 @@ from resource_management.libraries.functions.default import default
from resource_management.core.resources.system import Directory, File
from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate, Template
+from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config
from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME
@@ -121,6 +122,7 @@ def setup_logfeeder():
content=params.logfeeder_secure_log_content
)
+ generate_logfeeder_input_config('logsearch', Template("input.config-logsearch.json.j2", extra_imports=[default]))
if params.logsearch_solr_kerberos_enabled:
File(format("{logfeeder_jaas_file}"),
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 63b33e7..c6ad498 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -19,10 +19,12 @@ limitations under the License.
from resource_management.core.exceptions import Fail
from resource_management.core.resources.system import Directory, Execute, File
+from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate, Template
from resource_management.libraries.functions import solr_cloud_util
from resource_management.libraries.functions.decorator import retry
+from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config
from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME
@@ -137,6 +139,7 @@ def setup_logsearch():
Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")),
sudo=True
)
+ generate_logfeeder_input_config('logsearch', Template("input.config-logsearch.json.j2", extra_imports=[default]))
check_znode()
if params.security_enabled and not params.logsearch_use_external_solr:
diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
index ceec93f..6fc8f1e 100644
--- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
@@ -20,6 +20,7 @@ limitations under the License.
import os
from ambari_commons.constants import AMBARI_SUDO_BINARY
+from ambari_commons.constants import LOGFEEDER_CONF_DIR
from resource_management.libraries.script import Script
from resource_management.libraries.script.script import get_config_lock_file
from resource_management.libraries.functions import default
@@ -47,7 +48,7 @@ major_stack_version = get_major_version(stack_version_formatted)
service_name = config['serviceName']
# logsearch configuration
-logsearch_logfeeder_conf = "/usr/lib/ambari-logsearch-logfeeder/conf"
+logsearch_logfeeder_conf = LOGFEEDER_CONF_DIR
agent_cache_dir = config['agentLevelParams']['agentCacheDir']
service_package_folder = config['serviceLevelParams']['service_package_folder']
diff --git a/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py b/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
index 237466a..694872c 100644
--- a/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
+++ b/ambari-server/src/test/python/stacks/2.4/AMBARI_INFRA/test_infra_solr.py
@@ -20,6 +20,7 @@ limitations under the License.
from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate, StaticFile
from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.libraries.functions.default import default
from resource_management.libraries.script.config_dictionary import UnknownConfiguration
from mock.mock import MagicMock, call, patch
@@ -109,6 +110,16 @@ class TestInfraSolr(RMFTestCase):
group='root',
content = Template('infra-solr.conf.j2')
)
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-logsearch.json',
+ mode=0644,
+ content = Template('input.config-logsearch.json.j2', extra_imports=[default])
+ )
self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --create-znode --retry 30 --interval 5')
self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --remove-admin-handlers --collection hadoop_logs --retry 5 --interval 10')
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 3bc48ac..8ae0acd 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -21,6 +21,7 @@ limitations under the License.
import grp
from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate
+from resource_management.libraries.functions.default import default
class TestLogFeeder(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package"
@@ -104,6 +105,15 @@ class TestLogFeeder(RMFTestCase):
content=InlineTemplate('output-grok-filter'),
encoding='utf-8'
)
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-logsearch.json',
+ mode=0644,
+ content = Template('input.config-logsearch.json.j2', extra_imports=[default])
+ )
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py",
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index 89bc492..0c3d38c 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -19,6 +19,7 @@ limitations under the License.
'''
from stacks.utils.RMFTestCase import RMFTestCase, Template, InlineTemplate
+from resource_management.libraries.functions.default import default
class TestLogSearch(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "LOGSEARCH/0.5.0/package"
@@ -149,6 +150,18 @@ class TestLogSearch(RMFTestCase):
self.assertResourceCalled('Execute', ('chmod', '-R', 'ugo+r', '/usr/lib/ambari-logsearch-portal/conf/solr_configsets'),
sudo = True
)
+
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf',
+ create_parents = True,
+ cd_access = 'a',
+ mode = 0755
+ )
+
+ self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-logfeeder/conf/input.config-logsearch.json',
+ mode=0644,
+ content = Template('input.config-logsearch.json.j2', extra_imports=[default])
+ )
+
self.assertResourceCalled('Execute', 'ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 30 --interval 5')
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.