You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/08/22 13:36:56 UTC
[01/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)" [Forced Update!]
Repository: ambari
Updated Branches:
refs/heads/branch-dev-logsearch 09299e6dc -> a565ead6f (forced update)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/test/python/TestUpgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestUpgradeHelper.py b/ambari-server/src/test/python/TestUpgradeHelper.py
new file mode 100644
index 0000000..6da4507
--- /dev/null
+++ b/ambari-server/src/test/python/TestUpgradeHelper.py
@@ -0,0 +1,1028 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+from mock.mock import MagicMock, call
+from mock.mock import patch
+
+from unittest import TestCase
+import sys
+import os
+import unittest
+import upgradeHelper
+import json
+import copy
+from StringIO import StringIO
+
+
+class UpgradeCatalogFactoryMock(upgradeHelper.UpgradeCatalogFactory):
+ def __init__(self, data):
+ self._load(data)
+
+ def _load(self, data):
+ fn = StringIO(data)
+ with patch("__builtin__.open") as open_mock:
+ open_mock.return_value = fn
+ super(UpgradeCatalogFactoryMock, self)._load("")
+
+
+class TestUpgradeHelper(TestCase):
+ original_curl = None
+ out = None
+ catalog_from = "1.3"
+ catalog_to = "2.2"
+ catalog_cfg_type = "my type"
+ required_service = "TEST"
+ curl_response = "{}"
+ test_catalog = """{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "%s",
+ "target-version": "%s",
+ "options": {
+ "config-types": {
+ "%s": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "%s": {
+ "my property": {
+ "value": "my value",
+ "required-services": [\"%s\"]
+ }
+ }
+ },
+ "property-mapping": {
+ "my replace property": "my property 2"
+ }
+ }
+ ]
+ }
+ """
+
+ def setUp(self):
+ # replace original curl call to mock
+ self.test_catalog = self.test_catalog % (self.catalog_from, self.catalog_to,
+ self.catalog_cfg_type, self.catalog_cfg_type,
+ self.required_service)
+
+ self.original_curl = upgradeHelper.curl
+ upgradeHelper.curl = self.magic_curl
+
+ # mock logging methods
+ upgradeHelper.logging.getLogger = MagicMock()
+ upgradeHelper.logging.FileHandler = MagicMock()
+
+ self.out = StringIO()
+ sys.stdout = self.out
+
+ def magic_curl(self, *args, **kwargs):
+ resp = self.curl_response
+ self.curl_response = "{}"
+ if "parse" in kwargs and isinstance(resp, str) and kwargs["parse"] == True:
+ resp = json.loads(resp)
+ return resp
+
+ def tearDown(self):
+ sys.stdout = sys.__stdout__
+
+ @patch("optparse.OptionParser")
+ @patch("upgradeHelper.modify_configs")
+ @patch("__builtin__.open")
+ def test_ParseOptions(self, open_mock, modify_action_mock, option_parser_mock):
+ class options(object):
+ user = "test_user"
+ hostname = "127.0.0.1"
+ clustername = "test1"
+ password = "test_password"
+ upgrade_json = "catalog_file"
+ from_stack = "0.0"
+ to_stack = "1.3"
+ logfile = "test.log"
+ report = "report.txt"
+ https = False
+ port = "8080"
+ warnings = []
+ printonly = False
+
+ args = ["update-configs"]
+ modify_action_mock.return_value = MagicMock()
+ test_mock = MagicMock()
+ test_mock.parse_args = lambda: (options, args)
+ option_parser_mock.return_value = test_mock
+
+ upgradeHelper.main()
+
+ self.assertEqual("8080", upgradeHelper.Options.API_PORT)
+ self.assertEqual("http", upgradeHelper.Options.API_PROTOCOL)
+ self.assertEqual(1, modify_action_mock.call_count)
+ self.assertEqual({"user": options.user, "pass": options.password}, upgradeHelper.Options.API_TOKENS)
+ self.assertEqual(options.clustername, upgradeHelper.Options.CLUSTER_NAME)
+
+ def test_is_services_exists(self):
+ old_services = upgradeHelper.Options.SERVICES
+
+ upgradeHelper.Options.SERVICES = set(['TEST1', 'TEST2'])
+ actual_result = upgradeHelper.is_services_exists(['TEST1'])
+
+ # check for situation with two empty sets
+ upgradeHelper.Options.SERVICES = set()
+ actual_result_1 = upgradeHelper.is_services_exists([])
+
+ upgradeHelper.Options.SERVICES = old_services
+
+ self.assertEqual(True, actual_result)
+ self.assertEqual(True, actual_result_1)
+
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "isfile")
+ @patch("os.remove")
+ def test_write_mapping(self, remove_mock, isfile_mock, open_mock):
+ test_data = {
+ "test_field": "test_value"
+ }
+ test_result = json.dumps(test_data)
+ output = StringIO()
+ isfile_mock.return_value = True
+ open_mock.return_value = output
+
+ # execute testing function
+ upgradeHelper.write_mapping(test_data)
+
+ self.assertEquals(1, isfile_mock.call_count)
+ self.assertEquals(1, remove_mock.call_count)
+ self.assertEquals(1, open_mock.call_count)
+
+ # check for content
+ self.assertEquals(test_result, output.getvalue())
+
+ @patch("__builtin__.open")
+ @patch.object(os.path, "isfile")
+ def test_read_mapping(self, isfile_mock, open_mock):
+ test_data = {
+ "test_field": "test_value"
+ }
+ test_result = json.dumps(test_data)
+ isfile_mock.return_value = True
+ output = StringIO(test_result)
+ open_mock.return_value = output
+
+ # execute testing function
+ actual_mapping = upgradeHelper.read_mapping()
+
+ self.assertEquals(1, isfile_mock.call_count)
+ self.assertEquals(1, open_mock.call_count)
+
+ self.assertEquals(test_data, actual_mapping)
+
+ @patch.object(upgradeHelper, "curl")
+ @patch.object(upgradeHelper, "write_mapping")
+ def test_get_mr1_mapping(self, write_mapping_mock, curl_mock):
+ return_data = [
+ {
+ "host_components": [ # MAPREDUCE_CLIENT
+ {
+ "HostRoles": {
+ "host_name": "test.host.vm"
+ }
+ }
+ ]
+ },
+ {
+ "host_components": [ # JOBTRACKER
+ {
+ "HostRoles": {
+ "host_name": "test1.host.vm"
+ }
+ }
+ ]
+ },
+ {
+ "host_components": [ # TASKTRACKER
+ {
+ "HostRoles": {
+ "host_name": "test2.host.vm"
+ }
+ }
+ ]
+ },
+ {
+ "host_components": [ # HISTORYSERVER
+ {
+ "HostRoles": {
+ "host_name": "test3.host.vm"
+ }
+ }
+ ]
+ }
+ ]
+ expect_data = {
+ "MAPREDUCE_CLIENT": ["test.host.vm"],
+ "JOBTRACKER": ["test1.host.vm"],
+ "TASKTRACKER": ["test2.host.vm"],
+ "HISTORYSERVER": ["test3.host.vm"]
+ }
+
+ tricky_mock = MagicMock(side_effect=return_data)
+ curl_mock.side_effect = tricky_mock
+
+ # execute testing function
+ upgradeHelper.get_mr1_mapping()
+
+ self.assertEquals(write_mapping_mock.call_count, 1)
+ self.assertEquals(expect_data, write_mapping_mock.call_args[0][0])
+
+ @patch.object(upgradeHelper, "get_choice_string_input")
+ def test_get_YN_input(self, get_choice_string_input_mock):
+ yes = set(['yes', 'ye', 'y'])
+ no = set(['no', 'n'])
+
+ prompt = "test prompt"
+ default = "default value"
+
+ # execute testing function
+ upgradeHelper.get_YN_input(prompt, default)
+
+ expect_args = (prompt, default, yes, no)
+ self.assertEquals(expect_args, get_choice_string_input_mock.call_args[0])
+
+ @patch("__builtin__.raw_input")
+ def test_get_choice_string_input(self, raw_input_mock):
+ yes = set(['yes', 'ye', 'y'])
+ no = set(['no', 'n'])
+ input_answers = ["yes", "no", ""]
+ tricky_mock = MagicMock(side_effect=input_answers)
+ raw_input_mock.side_effect = tricky_mock
+ default = "default value"
+
+ expect_result = [True, False, default]
+ actual_result = []
+ for i in range(0, len(input_answers)):
+ actual_result.append(upgradeHelper.get_choice_string_input("test prompt", default, yes, no))
+
+ self.assertEquals(expect_result, actual_result)
+
+ @patch.object(upgradeHelper, "get_YN_input")
+ @patch.object(upgradeHelper, "read_mapping")
+ @patch.object(upgradeHelper, "curl")
+ def test_delete_mr(self, curl_mock, read_mapping_mock, get_YN_mock):
+ COMPONENT_URL_FORMAT = upgradeHelper.Options.CLUSTER_URL + '/hosts/%s/host_components/%s'
+ SERVICE_URL_FORMAT = upgradeHelper.Options.CLUSTER_URL + '/services/MAPREDUCE'
+ NON_CLIENTS = ["JOBTRACKER", "TASKTRACKER", "HISTORYSERVER"]
+ PUT_IN_DISABLED = {
+ "HostRoles": {
+ "state": "DISABLED"
+ }
+ }
+ mr_mapping = {
+ "MAPREDUCE_CLIENT": ["test.host.vm"],
+ "JOBTRACKER": ["test1.host.vm"],
+ "TASKTRACKER": ["test2.host.vm"],
+ "HISTORYSERVER": ["test3.host.vm"]
+ }
+ expected_curl_exec_args = []
+ for key, hosts in mr_mapping.items():
+ if key in NON_CLIENTS:
+ for host in hosts:
+ expected_curl_exec_args.append(
+ [
+ (COMPONENT_URL_FORMAT % (host, key),),
+ {
+ "request_type": "PUT",
+ "data": PUT_IN_DISABLED,
+ "validate": True
+ }
+ ]
+ )
+
+ expected_curl_exec_args.append(
+ [
+ (SERVICE_URL_FORMAT,),
+ {
+ "request_type": "DELETE",
+ "validate": True
+ }
+ ]
+ )
+
+ get_YN_mock.return_value = True
+ read_mapping_mock.return_value = mr_mapping
+
+ # execute testing function
+ upgradeHelper.delete_mr()
+
+ self.assertEqual(expected_curl_exec_args, curl_mock.call_args_list)
+
+ pass
+
+ @patch.object(upgradeHelper, "curl")
+ def test_get_cluster_stackname(self, curl_mock):
+ expected_result = "test version"
+ actual_result = ""
+ curl_mock.return_value = {
+ "Clusters": {
+ "version": expected_result
+ }
+ }
+
+ # execute testing function
+ actual_result = upgradeHelper.get_cluster_stackname()
+
+ self.assertEqual(expected_result, actual_result)
+
+ @patch.object(upgradeHelper, "curl")
+ def test_has_component_in_stack_def(self, curl_mock):
+ curl_mock.side_effect = MagicMock(side_effect=["", upgradeHelper.FatalException(1, "some reason")])
+
+ # execute testing function
+ result_ok = upgradeHelper.has_component_in_stack_def("-", "", "")
+ result_fail = upgradeHelper.has_component_in_stack_def("-", "", "")
+
+ self.assertEqual(True, result_ok)
+ self.assertEqual(False, result_fail)
+
+ @patch.object(upgradeHelper, "get_cluster_stackname")
+ @patch.object(upgradeHelper, "has_component_in_stack_def")
+ @patch.object(upgradeHelper, "read_mapping")
+ @patch.object(upgradeHelper, "curl")
+ def test_add_services(self, curl_mock, read_mapping_mock, has_component_mock, get_stack_name_mock):
+ host_mapping = {
+ "MAPREDUCE_CLIENT": ["test.host.vm"],
+ "JOBTRACKER": ["test1.host.vm"],
+ "TASKTRACKER": ["test2.host.vm"],
+ "HISTORYSERVER": ["test3.host.vm"]
+ }
+ SERVICE_URL_FORMAT = upgradeHelper.Options.CLUSTER_URL + '/services/{0}'
+ COMPONENT_URL_FORMAT = SERVICE_URL_FORMAT + '/components/{1}'
+ HOST_COMPONENT_URL_FORMAT = upgradeHelper.Options.CLUSTER_URL + '/hosts/{0}/host_components/{1}'
+ service_comp = {
+ "YARN": ["NODEMANAGER", "RESOURCEMANAGER", "YARN_CLIENT"],
+ "MAPREDUCE2": ["HISTORYSERVER", "MAPREDUCE2_CLIENT"]}
+ new_old_host_map = {
+ "NODEMANAGER": "TASKTRACKER",
+ "HISTORYSERVER": "HISTORYSERVER",
+ "RESOURCEMANAGER": "JOBTRACKER",
+ "YARN_CLIENT": "MAPREDUCE_CLIENT",
+ "MAPREDUCE2_CLIENT": "MAPREDUCE_CLIENT"}
+ get_stack_name_mock.return_value = ""
+ has_component_mock.return_value = False
+ read_mapping_mock.return_value = host_mapping
+ expected_curl_args = []
+
+ for service in service_comp.keys():
+ expected_curl_args.append([
+ (SERVICE_URL_FORMAT.format(service),),
+ {
+ "validate": True,
+ "request_type": "POST"
+ }
+ ])
+ for component in service_comp[service]:
+ expected_curl_args.append([
+ (COMPONENT_URL_FORMAT.format(service, component),),
+ {
+ "validate": True,
+ "request_type": "POST"
+ }
+ ])
+ for host in host_mapping[new_old_host_map[component]]:
+ expected_curl_args.append([
+ (HOST_COMPONENT_URL_FORMAT.format(host, component),),
+ {
+ "validate": True,
+ "request_type": "POST"
+ }
+ ])
+
+ # execute testing function
+ upgradeHelper.add_services()
+
+ self.assertEqual(expected_curl_args, curl_mock.call_args_list)
+
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ def test_coerce_tag(self, get_config_resp_all_mock):
+ test_catalog = """
+ {
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "1.0",
+ "target-version": "1.1",
+ "options": {
+ "config-types":{
+ "test": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "test": {
+ "test": "host1.com"
+ }
+ },
+ "property-mapping": {
+ "test":{
+ "map-to": "test-arr",
+ "coerce-to": "yaml-array"
+ }
+ }
+ }
+ ]
+ }
+ """
+ old_opt = upgradeHelper.Options.OPTIONS
+ options = lambda: ""
+ options.from_stack = "1.0"
+ options.to_stack = "1.1"
+ options.upgrade_json = ""
+
+ upgradeHelper.Options.OPTIONS = options
+ upgradeHelper.Options.SERVICES = [self.required_service]
+ get_config_resp_all_mock.return_value = {
+ "test": {
+ "properties": {}
+ }
+ }
+
+ ucf = UpgradeCatalogFactoryMock(test_catalog)
+ scf = upgradeHelper.ServerConfigFactory()
+
+ cfg = scf.get_config("test")
+ ucfg = ucf.get_catalog("1.0", "1.1")
+
+ cfg.merge(ucfg)
+ scf.process_mapping_transformations(ucfg)
+
+ upgradeHelper.Options.OPTIONS = old_opt
+
+ self.assertEqual(True, "test-arr" in cfg.properties)
+ self.assertEqual("['host1.com']", cfg.properties["test-arr"])
+
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ def test_override_tag(self, get_config_resp_all_mock):
+ test_catalog = """
+ {
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "1.0",
+ "target-version": "1.1",
+ "options": {
+ "config-types":{
+ "test": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "test": {
+ "test_property": {
+ "value": "host1.com",
+ "override": "no"
+ }
+
+ }
+ },
+ "property-mapping": {}
+ }
+ ]
+ }
+ """
+ old_opt = upgradeHelper.Options.OPTIONS
+ options = lambda: ""
+ options.from_stack = "1.0"
+ options.to_stack = "1.1"
+ options.upgrade_json = ""
+
+ upgradeHelper.Options.OPTIONS = options
+ upgradeHelper.Options.SERVICES = [self.required_service]
+ get_config_resp_all_mock.return_value = {
+ "test": {
+ "properties": {
+ "test_property": "test host"
+ }
+ }
+ }
+
+ ucf = UpgradeCatalogFactoryMock(test_catalog)
+ scf = upgradeHelper.ServerConfigFactory()
+
+ cfg = scf.get_config("test")
+ ucfg = ucf.get_catalog("1.0", "1.1")
+
+ cfg.merge(ucfg)
+ scf.process_mapping_transformations(ucfg)
+
+ upgradeHelper.Options.OPTIONS = old_opt
+
+ self.assertEqual(True, "test_property" in cfg.properties)
+ self.assertEqual("test host", cfg.properties["test_property"])
+
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ def test_replace_tag(self, get_config_resp_all_mock):
+ test_catalog = """
+ {
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "1.0",
+ "target-version": "1.1",
+ "options": {
+ "config-types":{
+ "test": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "test": {
+ "test": "host1.com"
+ }
+ },
+ "property-mapping": {
+ "test":{
+ "map-to": "test-arr",
+ "replace-from": "com",
+ "replace-to": "org"
+ }
+ }
+ }
+ ]
+ }
+ """
+ old_opt = upgradeHelper.Options.OPTIONS
+ options = lambda: ""
+ options.from_stack = "1.0"
+ options.to_stack = "1.1"
+ options.upgrade_json = ""
+
+ upgradeHelper.Options.OPTIONS = options
+ upgradeHelper.Options.SERVICES = [self.required_service]
+ get_config_resp_all_mock.return_value = {
+ "test": {
+ "properties": {}
+ }
+ }
+
+ ucf = UpgradeCatalogFactoryMock(test_catalog)
+ scf = upgradeHelper.ServerConfigFactory()
+
+ cfg = scf.get_config("test")
+ ucfg = ucf.get_catalog("1.0", "1.1")
+
+ cfg.merge(ucfg)
+ scf.process_mapping_transformations(ucfg)
+
+ upgradeHelper.Options.OPTIONS = old_opt
+
+ self.assertEqual(True, "test-arr" in cfg.properties)
+ self.assertEqual("host1.org", cfg.properties["test-arr"])
+
+ @patch.object(upgradeHelper, "curl")
+ @patch("time.time")
+ def test_update_config(self, time_mock, curl_mock):
+ time_pass = 2
+ config_type = "test config"
+ properties = {
+ "test property": "test value"
+ }
+ attributes = {
+ "test attribute": "attribute value"
+ }
+ expected_tag = "version" + str(int(time_pass * 1000))
+ properties_payload = {"Clusters": {"desired_config": {"type": config_type, "tag": expected_tag, "properties": properties}}}
+ time_mock.return_value = time_pass
+
+ expected_simple_result = (
+ (upgradeHelper.Options.CLUSTER_URL,),
+ {
+ "request_type": "PUT",
+ "data": copy.deepcopy(properties_payload),
+ "validate": True,
+ "soft_validation": True
+ }
+ )
+
+ properties_payload["Clusters"]["desired_config"]["properties_attributes"] = attributes
+ expected_complex_result = (
+ (upgradeHelper.Options.CLUSTER_URL,),
+ {
+ "request_type": "PUT",
+ "data": copy.deepcopy(properties_payload),
+ "validate": True,
+ "soft_validation": True
+ }
+ )
+
+ # execute testing function
+ upgradeHelper.update_config(properties, config_type)
+ simple_result = tuple(curl_mock.call_args)
+
+ upgradeHelper.update_config(properties, config_type, attributes)
+ complex_result = tuple(curl_mock.call_args)
+
+ self.assertEqual(expected_simple_result, simple_result)
+ self.assertEqual(expected_complex_result, complex_result)
+
+ @patch.object(upgradeHelper, "curl")
+ def test_get_zookeeper_quorum(self, curl_mock):
+ zoo_def_port = "2181"
+ return_curl_data = {
+ "host_components": [
+ {
+ "HostRoles": {
+ "host_name": "test.host.vm"
+ }
+ },
+ {
+ "HostRoles": {
+ "host_name": "test.host.vm"
+ }
+ }
+ ]
+ }
+
+ curl_mock.return_value = copy.deepcopy(return_curl_data)
+
+ # build zookeeper quorum string from return_curl_data and remove trailing comas
+ expected_result = reduce(
+ lambda x, y: x + "%s:%s," % (y["HostRoles"]["host_name"], zoo_def_port),
+ return_curl_data["host_components"],
+ '' # initializer
+ ).rstrip(',')
+
+ # execute testing function
+ actual_result = upgradeHelper.get_zookeeper_quorum()
+
+ self.assertEqual(expected_result, actual_result)
+
+ @patch.object(upgradeHelper, "curl")
+ def test_get_tez_history_url_base(self, curl_mock):
+ return_curl_data = {
+ 'href': 'http://127.0.0.1:8080/api/v1/views/TEZ',
+ 'ViewInfo': {'view_name': 'TEZ'},
+ 'versions': [
+ {
+ 'ViewVersionInfo': {
+ 'view_name': 'TEZ',
+ 'version': '0.7.0.2.3.0.0-1319'
+ },
+ 'href': 'http://127.0.0.1:8080/api/v1/views/TEZ/versions/0.7.0.2.3.0.0-1319'
+ }
+ ]
+ }
+
+ curl_mock.return_value = copy.deepcopy(return_curl_data)
+
+ # build zookeeper quorum string from return_curl_data and remove trailing comas
+ expected_result = "http://127.0.0.1:8080/#/main/views/TEZ/0.7.0.2.3.0.0-1319/TEZ_CLUSTER_INSTANCE"
+
+ # execute testing function
+ actual_result = upgradeHelper.get_tez_history_url_base()
+
+ self.assertEqual(expected_result, actual_result)
+
+ @patch.object(upgradeHelper, "curl")
+ def test_get_ranger_xaaudit_hdfs_destination_directory(self, curl_mock):
+ return_curl_data = {
+ "host_components": [
+ {
+ "HostRoles": {
+ "host_name": "test.host.vm"
+ }
+ }
+ ]
+ }
+
+ curl_mock.return_value = copy.deepcopy(return_curl_data)
+
+ # build zookeeper quorum string from return_curl_data and remove trailing comas
+ expected_result = "hdfs://test.host.vm:8020/ranger/audit"
+
+ # execute testing function
+ actual_result = upgradeHelper.get_ranger_xaaudit_hdfs_destination_directory()
+
+ self.assertEqual(expected_result, actual_result)
+
+
+ @patch.object(upgradeHelper, "curl")
+ def test_get_config_resp_all(self, curl_mock):
+ cfg_type = "my type"
+ cfg_tag = "my tag"
+ cfg_properties = {
+ "my property": "property value"
+ }
+ curl_resp = [
+ {
+ 'Clusters': {
+ 'desired_configs': {
+ cfg_type: {
+ "tag": cfg_tag
+ }
+ }
+ }
+ },
+ {
+ "items": [
+ {
+ "type": cfg_type,
+ "tag": cfg_tag,
+ "properties": cfg_properties
+ }
+ ]
+ }
+ ]
+
+ expected_result = {
+ cfg_type: {
+ "properties": cfg_properties,
+ "tag": cfg_tag
+ }
+ }
+ curl_mock.side_effect = MagicMock(side_effect=curl_resp)
+
+ # execute testing function
+ actual_result = upgradeHelper.get_config_resp_all()
+
+ self.assertEquals(expected_result, actual_result)
+ pass
+
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ @patch("os.mkdir")
+ @patch("os.path.exists")
+ @patch("__builtin__.open")
+ def test_backup_configs(self, open_mock, os_path_exists_mock, mkdir_mock, get_config_resp_all_mock):
+ data = {
+ self.catalog_cfg_type: {
+ "properties": {
+ "test-property": "value"
+ },
+ "tag": "version1"
+ }
+ }
+ os_path_exists_mock.return_value = False
+ get_config_resp_all_mock.return_value = data
+ expected = json.dumps(data[self.catalog_cfg_type]["properties"], indent=4)
+ stream = StringIO()
+ m = MagicMock()
+ m.__enter__.return_value = stream
+ open_mock.return_value = m
+
+ # execute testing function
+ upgradeHelper.backup_configs(self.catalog_cfg_type)
+
+ self.assertEqual(expected, stream.getvalue())
+
+ @patch.object(upgradeHelper, "curl")
+ def test_install_services(self, curl_mock):
+ expected_args = (
+ (
+ ('http://127.0.0.1:8080/api/v1/clusters/test1/services/MAPREDUCE2',),
+ {
+ 'request_type': 'PUT',
+ 'data': {
+ 'RequestInfo': {
+ 'context': 'Install MapReduce2'
+ },
+ 'Body': {
+ 'ServiceInfo': {
+ 'state': 'INSTALLED'
+ }
+ }
+ },
+ 'validate': True
+ }
+ ),
+ (
+ ('http://127.0.0.1:8080/api/v1/clusters/test1/services/YARN',),
+ {
+ 'request_type': 'PUT',
+ 'data': {
+ 'RequestInfo': {
+ 'context': 'Install YARN'
+ },
+ 'Body': {
+ 'ServiceInfo': {
+ 'state': 'INSTALLED'
+ }
+ }
+ },
+ 'validate': True
+ }
+ )
+ )
+
+ # execute testing function
+ upgradeHelper.install_services()
+
+ self.assertEqual(2, curl_mock.call_count)
+ for i in range(0, 1):
+ self.assertEqual(expected_args[i], tuple(curl_mock.call_args_list[i]))
+
+ def test_configuration_diff_analyze(self):
+ in_data = {
+ self.catalog_cfg_type: [
+ {
+ 'catalog_item': {
+ 'value': 'my value'
+ },
+ 'property': 'my property',
+ 'actual_value': 'my value',
+ 'catalog_value': 'my value'
+ }
+ ]
+ }
+
+ expected_result = {
+ 'my type': {
+ 'fail': {
+ 'count': 0,
+ 'items': []
+ },
+ 'total': {
+ 'count': 1,
+ 'items': []
+ },
+ 'skipped': {
+ 'count': 0,
+ 'items': []
+ },
+ 'ok': {
+ 'count': 1,
+ 'items': [
+ {
+ 'catalog_item': {
+ 'value': 'my value'
+ },
+ 'property': 'my property',
+ 'actual_value': 'my value',
+ 'catalog_value': 'my value'
+ }
+ ]
+ }
+ }
+ }
+
+ # execute testing function
+ actual_result = upgradeHelper.configuration_diff_analyze(in_data)
+
+ self.assertEqual(expected_result, actual_result)
+
+ @patch.object(upgradeHelper, "UpgradeCatalogFactory", autospec=True)
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ @patch.object(upgradeHelper, "configuration_item_diff")
+ @patch.object(upgradeHelper, "configuration_diff_analyze")
+ @patch("__builtin__.open")
+ def test_verify_configuration(self, open_mock, configuration_diff_analyze_mock, configuration_item_diff_mock,
+ get_config_resp_all_mock, upgradecatalogfactory_mock):
+ old_opt = upgradeHelper.Options.OPTIONS
+ options = lambda: ""
+ options.from_stack = self.catalog_from
+ options.to_stack = self.catalog_to
+ options.upgrade_json = ""
+
+ upgradeHelper.Options.OPTIONS = options
+ upgradeHelper.Options.SERVICES = [self.required_service]
+ upgradecatalogfactory_mock.return_value = UpgradeCatalogFactoryMock(self.test_catalog)
+ get_config_resp_all_mock.return_value = {
+ self.catalog_cfg_type: {
+ "properties": {}
+ }
+ }
+
+ # execute testing function
+ upgradeHelper.verify_configuration()
+
+ upgradeHelper.Options.OPTIONS = old_opt
+
+ self.assertEqual(1, get_config_resp_all_mock.call_count)
+ self.assertEqual(1, configuration_item_diff_mock.call_count)
+ self.assertEqual(1, configuration_diff_analyze_mock.call_count)
+ self.assertEqual(1, open_mock.call_count)
+
+ def test_report_formatter(self):
+ file = StringIO()
+ cfg_item = self.catalog_cfg_type
+ analyzed_list = {
+ 'fail': {
+ 'count': 1,
+ 'items': [
+ {
+ 'catalog_item': {
+ 'value': 'my value'
+ },
+ 'property': 'my property',
+ 'actual_value': 'my value 1',
+ 'catalog_value': 'my value'
+ }
+ ]
+ },
+ 'total': {
+ 'count': 1,
+ 'items': []
+ },
+ 'skipped': {
+ 'count': 0,
+ 'items': []
+ },
+ 'ok': {
+ 'count': 0,
+ 'items': []
+ }
+ }
+
+ expected_output = "Configuration item my type: property \"my property\" is set to \"my value 1\", but should be set to \"my value\"\n"
+
+ # execute testing function
+ upgradeHelper.report_formatter(file, cfg_item, analyzed_list)
+
+ self.assertEqual(expected_output, file.getvalue())
+
+ @patch.object(upgradeHelper, "get_config_resp_all")
+ def test_conditional_replace(self, get_config_resp_all_mock):
+ test_catalog = """
+ {
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "1.0",
+ "target-version": "1.1",
+ "options": {
+ "config-types":{
+ "test": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "test": {
+ "test": {
+ "value": "10",
+ "value-required": "-1"
+ },
+ "test2": {
+ "value": "10",
+ "value-required": "-2"
+ }
+ }
+ },
+ "property-mapping": {
+ }
+ }
+ ]
+ }
+ """
+
+ expected_properties = {"test":"10", "test2":"15"}
+
+ old_opt = upgradeHelper.Options.OPTIONS
+ options = lambda: ""
+ options.from_stack = "1.0"
+ options.to_stack = "1.1"
+ options.upgrade_json = ""
+
+ upgradeHelper.Options.OPTIONS = options
+ upgradeHelper.Options.SERVICES = [self.required_service]
+ get_config_resp_all_mock.return_value = {
+ "test": {
+ "properties": {"test":"-1", "test2":"15"}
+ }
+ }
+
+ ucf = UpgradeCatalogFactoryMock(test_catalog)
+ scf = upgradeHelper.ServerConfigFactory()
+
+ cfg = scf.get_config("test")
+ ucfg = ucf.get_catalog("1.0", "1.1")
+
+ cfg.merge(ucfg)
+ upgradeHelper.Options.OPTIONS = old_opt
+
+ self.assertEqual(expected_properties, cfg.properties)
+
+if __name__ == "__main__":
+ unittest.main()
[21/55] [abbrv] ambari git commit: AMBARI-18206. trunk build failed
due to upgradeHelper.py not found (dgrinenko via dlysnichenko)
Posted by ol...@apache.org.
AMBARI-18206. trunk build failed due to upgradeHelper.py not found (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/356feed9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/356feed9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/356feed9
Branch: refs/heads/branch-dev-logsearch
Commit: 356feed94f16b1cc78ca4f407cd54828effd3b53
Parents: 5a943f4
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Aug 19 11:32:51 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Aug 19 11:33:12 2016 +0300
----------------------------------------------------------------------
ambari-server/src/main/assemblies/server.xml | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/356feed9/ambari-server/src/main/assemblies/server.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/assemblies/server.xml b/ambari-server/src/main/assemblies/server.xml
index 1560d8d..9252c56 100644
--- a/ambari-server/src/main/assemblies/server.xml
+++ b/ambari-server/src/main/assemblies/server.xml
@@ -300,11 +300,6 @@
</file>
<file>
<fileMode>755</fileMode>
- <source>src/main/python/upgradeHelper.py</source>
- <outputDirectory>/var/lib/ambari-server/resources/scripts</outputDirectory>
- </file>
- <file>
- <fileMode>755</fileMode>
<source>src/main/resources/stacks/stack_advisor.py</source>
<outputDirectory>/var/lib/ambari-server/resources/stacks</outputDirectory>
</file>
[37/55] [abbrv] ambari git commit: AMBARI-18219. Ambari should use
oozied.sh for stopping oozie so that optional catalina args can be provided -
fixing a typo (Venkat Ranganathan via smohanty)
Posted by ol...@apache.org.
AMBARI-18219. Ambari should use oozied.sh for stopping oozie so that optional catalina args can be provided - fixing a typo (Venkat Ranganathan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/967ce4be
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/967ce4be
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/967ce4be
Branch: refs/heads/branch-dev-logsearch
Commit: 967ce4be73bd3f1fe553e6930a70ddaf5e6acd2a
Parents: 5eb6946
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Aug 20 21:20:09 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Aug 20 21:21:05 2016 -0700
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie_service.py | 2 +-
.../src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/967ce4be/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 8371ed1..8d80638 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -158,7 +158,7 @@ def oozie_service(action = 'start', upgrade_type=None):
raise
elif action == 'stop':
- stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 force")
+ stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 -force")
try:
# stop oozie
http://git-wip-us.apache.org/repos/asf/ambari/blob/967ce4be/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index db7eca3..b0cc2e9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -576,7 +576,7 @@ class TestOozieServer(RMFTestCase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 force',
+ self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 -force',
environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
user = 'oozie',
@@ -706,7 +706,7 @@ class TestOozieServer(RMFTestCase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 force',
+ self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 -force',
environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
user = 'oozie',
[24/55] [abbrv] ambari git commit: AMBARI-18211. No config group
selected (akovalenko)
Posted by ol...@apache.org.
AMBARI-18211. No config group selected (akovalenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/407c4525
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/407c4525
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/407c4525
Branch: refs/heads/branch-dev-logsearch
Commit: 407c452517f04d05ddeb0d290adb48b500a5ead4
Parents: 9ad1966
Author: Aleksandr Kovalenko <ak...@hortonworks.com>
Authored: Fri Aug 19 17:23:12 2016 +0300
Committer: Aleksandr Kovalenko <ak...@hortonworks.com>
Committed: Fri Aug 19 17:23:12 2016 +0300
----------------------------------------------------------------------
ambari-web/app/mixins/common/configs/configs_loader.js | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/407c4525/ambari-web/app/mixins/common/configs/configs_loader.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/configs_loader.js b/ambari-web/app/mixins/common/configs/configs_loader.js
index 83e4837..87dec7f 100644
--- a/ambari-web/app/mixins/common/configs/configs_loader.js
+++ b/ambari-web/app/mixins/common/configs/configs_loader.js
@@ -125,9 +125,10 @@ App.ConfigsLoader = Em.Mixin.create(App.GroupsMappingMixin, {
*/
loadCurrentVersionsSuccess: function (data, opt, params) {
var self = this;
- var serviceGroups = App.ServiceConfigGroup.find().filterProperty('serviceName', this.get('content.serviceName'));
+ var serviceGroups;
App.configGroupsMapper.map(data, true, params.serviceNames.split(','));
this.loadConfigGroups(params.serviceNames.split(',')).done(function () {
+ serviceGroups = App.ServiceConfigGroup.find().filterProperty('serviceName', self.get('content.serviceName'));
if (self.get('isHostsConfigsPage')) {
self.set('selectedConfigGroup', serviceGroups.find(function (cg) {
return !cg.get('isDefault') && cg.get('hosts').contains(self.get('host.hostName'));
[46/55] [abbrv] ambari git commit: AMBARI-18079. Update the date
range picker with the latest version i.e 2.1.24 (Dharmesh Makwana via
oleewere)
Posted by ol...@apache.org.
AMBARI-18079. Update the date range picker with the latest version i.e 2.1.24 (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74b1952f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74b1952f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74b1952f
Branch: refs/heads/branch-dev-logsearch
Commit: 74b1952fff351943d59307518b99c160eece2ed1
Parents: 650e6c4
Author: oleewere <ol...@gmail.com>
Authored: Thu Aug 18 13:17:30 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:02 2016 +0200
----------------------------------------------------------------------
.../daterangepicker/css/daterangepicker.css | 555 +++++++++++--------
.../other/daterangepicker/js/daterangepicker.js | 552 +++++++++++++-----
.../scripts/views/common/DatePickerLayout.js | 23 +-
.../scripts/views/graphs/GraphLayoutView.js | 3 +-
.../troubleshoot/TroubleShootLayoutView.js | 2 +-
.../src/main/webapp/styles/style.css | 11 +-
6 files changed, 736 insertions(+), 410 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/74b1952f/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/css/daterangepicker.css
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/css/daterangepicker.css b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/css/daterangepicker.css
index f2ff53d..7f1ef54 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/css/daterangepicker.css
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/css/daterangepicker.css
@@ -1,336 +1,415 @@
-/**
-* A stylesheet for use with Bootstrap 3.x
-* @author: Dan Grossman http://www.dangrossman.info/
-* @copyright: Copyright (c) 2012-2015 Dan Grossman. All rights reserved.
-* @license: Licensed under the MIT license. See http://www.opensource.org/licenses/mit-license.php
-* @website: https://www.improvely.com/
-*/
+.daterangepicker {
+ position: absolute;
+ color: inherit;
+ background: #fff;
+ border-radius: 4px;
+ width: 278px;
+ padding: 4px;
+ margin-top: 1px;
+ top: 100px;
+ left: 20px;
+ /* Calendars */
+}
+
+.daterangepicker:before,
+.daterangepicker:after {
+ position: absolute;
+ display: inline-block;
+ border-bottom-color: rgba(0, 0, 0, 0.2);
+ content: '';
+}
-/* Container Appearance */
+.daterangepicker:before {
+ top: -7px;
+ border-right: 7px solid transparent;
+ border-left: 7px solid transparent;
+ border-bottom: 7px solid #ccc;
+}
-.daterangepicker {
- position: absolute;
- background: #fff;
- top: 100px;
- left: 20px;
- padding: 4px;
- margin-top: 1px;
- border-radius: 4px;
+.daterangepicker:after {
+ top: -6px;
+ border-right: 6px solid transparent;
+ border-bottom: 6px solid #fff;
+ border-left: 6px solid transparent;
}
.daterangepicker.opensleft:before {
- position: absolute;
- top: -7px;
- right: 9px;
- display: inline-block;
- border-right: 7px solid transparent;
- border-bottom: 7px solid #ccc;
- border-left: 7px solid transparent;
- border-bottom-color: rgba(0, 0, 0, 0.2);
- content: '';
+ right: 9px;
}
.daterangepicker.opensleft:after {
- position: absolute;
- top: -6px;
- right: 10px;
- display: inline-block;
- border-right: 6px solid transparent;
- border-bottom: 6px solid #fff;
- border-left: 6px solid transparent;
- content: '';
+ right: 10px;
}
.daterangepicker.openscenter:before {
- position: absolute;
- top: -7px;
- left: 0;
- right: 0;
- width: 0;
- margin-left: auto;
- margin-right: auto;
- display: inline-block;
- border-right: 7px solid transparent;
- border-bottom: 7px solid #ccc;
- border-left: 7px solid transparent;
- border-bottom-color: rgba(0, 0, 0, 0.2);
- content: '';
+ left: 0;
+ right: 0;
+ width: 0;
+ margin-left: auto;
+ margin-right: auto;
}
.daterangepicker.openscenter:after {
- position: absolute;
- top: -6px;
- left: 0;
- right: 0;
- width: 0;
- margin-left: auto;
- margin-right: auto;
- display: inline-block;
- border-right: 6px solid transparent;
- border-bottom: 6px solid #fff;
- border-left: 6px solid transparent;
- content: '';
+ left: 0;
+ right: 0;
+ width: 0;
+ margin-left: auto;
+ margin-right: auto;
}
.daterangepicker.opensright:before {
- position: absolute;
- top: -7px;
- left: 9px;
- display: inline-block;
- border-right: 7px solid transparent;
- border-bottom: 7px solid #ccc;
- border-left: 7px solid transparent;
- border-bottom-color: rgba(0, 0, 0, 0.2);
- content: '';
+ left: 9px;
}
.daterangepicker.opensright:after {
- position: absolute;
- top: -6px;
- left: 10px;
- display: inline-block;
- border-right: 6px solid transparent;
- border-bottom: 6px solid #fff;
- border-left: 6px solid transparent;
- content: '';
+ left: 10px;
}
-.daterangepicker.dropup{
- margin-top: -5px;
-}
-.daterangepicker.dropup:before{
- top: initial;
- bottom:-7px;
- border-bottom: initial;
- border-top: 7px solid #ccc;
-}
-.daterangepicker.dropup:after{
- top: initial;
- bottom:-6px;
- border-bottom: initial;
- border-top: 6px solid #fff;
+.daterangepicker.dropup {
+ margin-top: -5px;
}
-.daterangepicker.dropdown-menu {
- max-width: none;
- z-index: 3000;
+.daterangepicker.dropup:before {
+ top: initial;
+ bottom: -7px;
+ border-bottom: initial;
+ border-top: 7px solid #ccc;
}
-.daterangepicker .ranges, .daterangepicker .calendar {
- float: left;
+.daterangepicker.dropup:after {
+ top: initial;
+ bottom: -6px;
+ border-bottom: initial;
+ border-top: 6px solid #fff;
}
-.daterangepicker.single .ranges, .daterangepicker.single .calendar {
- float: none;
+.daterangepicker.dropdown-menu {
+ max-width: none;
+ z-index: 3001;
}
-.daterangepicker .ranges {
- margin: 4px;
- text-align: left;
+.daterangepicker.single .ranges,
+.daterangepicker.single .calendar {
+ float: none;
}
-.daterangepicker .calendar {
- display: none;
- max-width: 270px;
+.daterangepicker.show-calendar .calendar {
+ display: block;
}
-.daterangepicker.show-calendar .calendar {
- display: block;
+.daterangepicker .calendar {
+ display: none;
+ max-width: 270px;
+ margin: 4px;
}
.daterangepicker .calendar.single .calendar-table {
- border: none;
+ border: none;
}
-/* Calendars */
-
-.daterangepicker .calendar th, .daterangepicker .calendar td {
- white-space: nowrap;
- text-align: center;
- min-width: 32px;
+.daterangepicker .calendar th,
+.daterangepicker .calendar td {
+ white-space: nowrap;
+ text-align: center;
+ min-width: 32px;
}
.daterangepicker .calendar-table {
- border: 1px solid #ddd;
- padding: 4px;
- border-radius: 4px;
- background: #fff;
-}
-
-.daterangepicker .calendar.left .calendar-table {
- border-right: none;
- border-top-right-radius: 0;
- border-bottom-right-radius: 0;
-}
-
-.daterangepicker .calendar.right .calendar-table {
- border-left: none;
- border-top-left-radius: 0;
- border-bottom-left-radius: 0;
-}
-
-.daterangepicker .calendar.left {
- margin: 4px 0 4px 4px;
-}
-
-.daterangepicker .calendar.right {
- margin: 4px 4px 4px 0;
-}
-
-.daterangepicker .calendar.left .calendar-table {
- padding-right: 12px;
+ border: 1px solid #fff;
+ padding: 4px;
+ border-radius: 4px;
+ background: #fff;
}
.daterangepicker table {
- width: 100%;
- margin: 0;
+ width: 100%;
+ margin: 0;
}
-.daterangepicker td, .daterangepicker th {
- text-align: center;
- width: 20px;
- height: 20px;
- border-radius: 4px;
- white-space: nowrap;
- cursor: pointer;
+.daterangepicker td,
+.daterangepicker th {
+ text-align: center;
+ width: 20px;
+ height: 20px;
+ border-radius: 4px;
+ border: 1px solid transparent;
+ white-space: nowrap;
+ cursor: pointer;
}
-.daterangepicker td.off {
- color: #999;
+.daterangepicker td.available:hover,
+.daterangepicker th.available:hover {
+ background-color: #eee;
+ border-color: transparent;
+ color: inherit;
}
-.daterangepicker td.disabled, .daterangepicker option.disabled {
- color: #999;
+.daterangepicker td.week,
+.daterangepicker th.week {
+ font-size: 80%;
+ color: #ccc;
}
-.daterangepicker td.available:hover, .daterangepicker th.available:hover {
- background: #eee;
+.daterangepicker td.off,
+.daterangepicker td.off.in-range,
+.daterangepicker td.off.start-date,
+.daterangepicker td.off.end-date {
+ background-color: #fff;
+ border-color: transparent;
+ color: #999;
}
.daterangepicker td.in-range {
- background: #ebf4f8;
- border-radius: 0;
+ background-color: #ebf4f8;
+ border-color: transparent;
+ color: #000;
+ border-radius: 0;
}
.daterangepicker td.start-date {
- border-radius: 4px 0 0 4px;
+ border-radius: 4px 0 0 4px;
}
.daterangepicker td.end-date {
- border-radius: 0 4px 4px 0;
+ border-radius: 0 4px 4px 0;
}
.daterangepicker td.start-date.end-date {
- border-radius: 4px;
-}
-
-.daterangepicker td.active, .daterangepicker td.active:hover {
- background-color: #357ebd;
- border-color: #3071a9;
- color: #fff;
+ border-radius: 4px;
}
-.daterangepicker td.week, .daterangepicker th.week {
- font-size: 80%;
- color: #ccc;
+.daterangepicker td.active,
+.daterangepicker td.active:hover {
+ background-color: #357ebd;
+ border-color: transparent;
+ color: #fff;
}
-.daterangepicker select.monthselect, .daterangepicker select.yearselect {
- font-size: 12px;
- padding: 1px;
- height: auto;
- margin: 0;
- cursor: default;
+.daterangepicker th.month {
+ width: auto;
}
-.daterangepicker select.monthselect {
- margin-right: 2%;
- width: 56%;
+.daterangepicker td.disabled,
+.daterangepicker option.disabled {
+ color: #999;
+ cursor: not-allowed;
+ text-decoration: line-through;
}
+.daterangepicker select.monthselect,
.daterangepicker select.yearselect {
- width: 40%;
+ font-size: 12px;
+ padding: 1px;
+ height: auto;
+ margin: 0;
+ cursor: default;
}
-.daterangepicker select.hourselect, .daterangepicker select.minuteselect, .daterangepicker select.secondselect, .daterangepicker select.ampmselect {
- width: 50px;
- margin-bottom: 0;
+.daterangepicker select.monthselect {
+ margin-right: 2%;
+ width: 56%;
}
-.daterangepicker th.month {
- width: auto;
+.daterangepicker select.yearselect {
+ width: 40%;
}
-/* Text Input Above Each Calendar */
-
-.daterangepicker .input-mini {
- border: 1px solid #ccc;
- border-radius: 4px;
- color: #555;
- display: block;
- height: 30px;
- line-height: 30px;
- vertical-align: middle;
- margin: 0 0 5px 0;
- padding: 0 6px 0 28px;
- width: 100%;
+.daterangepicker select.hourselect,
+.daterangepicker select.minuteselect,
+.daterangepicker select.secondselect,
+.daterangepicker select.ampmselect {
+ width: 50px;
+ margin-bottom: 0;
}
-.daterangepicker .daterangepicker_input i {
- position: absolute;
- left: 8px;
- top: 8px;
+.daterangepicker .input-mini {
+ border: 1px solid #ccc;
+ border-radius: 4px;
+ color: #555;
+ height: 30px;
+ line-height: 30px;
+ display: block;
+ vertical-align: middle;
+ margin: 0 0 5px 0;
+ padding: 0 6px 0 28px;
+ width: 100%;
}
-.daterangepicker .left .daterangepicker_input {
- padding-right: 12px;
+.daterangepicker .input-mini.active {
+ border: 1px solid #08c;
+ border-radius: 4px;
}
.daterangepicker .daterangepicker_input {
- position: relative;
-}
-
-/* Time Picker */
-
-.daterangepicker .calendar-time {
- text-align: center;
- margin: 5px auto;
- line-height: 30px;
- position: relative;
- padding-left: 28px;
+ position: relative;
}
-.daterangepicker .calendar-time select.disabled {
- color: #ccc;
- cursor: not-allowed;
+.daterangepicker .daterangepicker_input i {
+ position: absolute;
+ left: 8px;
+ top: 8px;
}
-/* Predefined Ranges */
-
-.daterangepicker .ranges {
- font-size: 11px;
+.daterangepicker.rtl .input-mini {
+ padding-right: 28px;
+ padding-left: 6px;
}
-.daterangepicker .ranges ul {
- list-style: none;
- margin: 0;
- padding: 0;
- width: 160px;
+.daterangepicker.rtl .daterangepicker_input i {
+ left: auto;
+ right: 8px;
}
-.daterangepicker .ranges li {
- font-size: 13px;
- background: #f5f5f5;
- border: 1px solid #f5f5f5;
- color: #08c;
- padding: 3px 12px;
- margin-bottom: 8px;
- border-radius: 5px;
- cursor: pointer;
+.daterangepicker .calendar-time {
+ text-align: center;
+ margin: 5px auto;
+ line-height: 30px;
+ position: relative;
+ padding-left: 28px;
}
-.daterangepicker .ranges li.active, .daterangepicker .ranges li:hover {
- background: #08c;
- border: 1px solid #08c;
- color: #fff;
+.daterangepicker .calendar-time select.disabled {
+ color: #ccc;
+ cursor: not-allowed;
+}
+
+.ranges {
+ font-size: 11px;
+ float: none;
+ margin: 4px;
+ text-align: left;
+}
+
+.ranges ul {
+ list-style: none;
+ margin: 0 auto;
+ padding: 0;
+ width: 100%;
+}
+
+.ranges li {
+ font-size: 13px;
+ background: #f5f5f5;
+ border: 1px solid #f5f5f5;
+ border-radius: 4px;
+ color: #08c;
+ padding: 3px 12px;
+ margin-bottom: 8px;
+ cursor: pointer;
+}
+
+.ranges li:hover {
+ background: #08c;
+ border: 1px solid #08c;
+ color: #fff;
+}
+
+.ranges li.active {
+ background: #08c;
+ border: 1px solid #08c;
+ color: #fff;
+}
+
+
+/* Larger Screen Styling */
+
+@media (min-width: 564px) {
+ .daterangepicker {
+ width: auto;
+ }
+ .daterangepicker .ranges ul {
+ width: 160px;
+ }
+ .daterangepicker.single .ranges ul {
+ width: 100%;
+ }
+ .daterangepicker.single .calendar.left {
+ clear: none;
+ }
+ .daterangepicker.single.ltr .ranges,
+ .daterangepicker.single.ltr .calendar {
+ float: left;
+ }
+ .daterangepicker.single.rtl .ranges,
+ .daterangepicker.single.rtl .calendar {
+ float: right;
+ }
+ .daterangepicker.ltr {
+ direction: ltr;
+ text-align: left;
+ }
+ .daterangepicker.ltr .calendar.left {
+ clear: left;
+ margin-right: 0;
+ }
+ .daterangepicker.ltr .calendar.left .calendar-table {
+ border-right: none;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 0;
+ }
+ .daterangepicker.ltr .calendar.right {
+ margin-left: 0;
+ }
+ .daterangepicker.ltr .calendar.right .calendar-table {
+ border-left: none;
+ border-top-left-radius: 0;
+ border-bottom-left-radius: 0;
+ }
+ .daterangepicker.ltr .left .daterangepicker_input {
+ padding-right: 12px;
+ }
+ .daterangepicker.ltr .calendar.left .calendar-table {
+ padding-right: 12px;
+ }
+ .daterangepicker.ltr .ranges,
+ .daterangepicker.ltr .calendar {
+ float: left;
+ }
+ .daterangepicker.rtl {
+ direction: rtl;
+ text-align: right;
+ }
+ .daterangepicker.rtl .calendar.left {
+ clear: right;
+ margin-left: 0;
+ }
+ .daterangepicker.rtl .calendar.left .calendar-table {
+ border-left: none;
+ border-top-left-radius: 0;
+ border-bottom-left-radius: 0;
+ }
+ .daterangepicker.rtl .calendar.right {
+ margin-right: 0;
+ }
+ .daterangepicker.rtl .calendar.right .calendar-table {
+ border-right: none;
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 0;
+ }
+ .daterangepicker.rtl .left .daterangepicker_input {
+ padding-left: 12px;
+ }
+ .daterangepicker.rtl .calendar.left .calendar-table {
+ padding-left: 12px;
+ }
+ .daterangepicker.rtl .ranges,
+ .daterangepicker.rtl .calendar {
+ text-align: right;
+ float: right;
+ }
+}
+
+@media (min-width: 730px) {
+ .daterangepicker .ranges {
+ width: auto;
+ }
+ .daterangepicker.ltr .ranges {
+ float: left;
+ }
+ .daterangepicker.rtl .ranges {
+ float: right;
+ }
+ .daterangepicker .calendar.left {
+ clear: none !important;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/74b1952f/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/js/daterangepicker.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/js/daterangepicker.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/js/daterangepicker.js
index 778af8d..04798c6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/js/daterangepicker.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/libs/other/daterangepicker/js/daterangepicker.js
@@ -1,37 +1,31 @@
/**
-* @version: 2.0
+* @version: 2.1.24
* @author: Dan Grossman http://www.dangrossman.info/
-* @copyright: Copyright (c) 2012-2015 Dan Grossman. All rights reserved.
+* @copyright: Copyright (c) 2012-2016 Dan Grossman. All rights reserved.
* @license: Licensed under the MIT license. See http://www.opensource.org/licenses/mit-license.php
* @website: https://www.improvely.com/
*/
-
-(function(root, factory) {
-
- if (typeof define === 'function' && define.amd) {
- define(['moment', 'jquery', 'exports'], function(momentjs, $, exports) {
- root.daterangepicker = factory(root, exports, momentjs, $);
- });
-
- } else if (typeof exports !== 'undefined') {
- var momentjs = require('moment');
- var jQuery;
- try {
- jQuery = require('jquery');
- } catch (err) {
- jQuery = window.jQuery;
- if (!jQuery) throw new Error('jQuery dependnecy not found');
+// Follow the UMD template https://github.com/umdjs/umd/blob/master/templates/returnExportsGlobal.js
+(function (root, factory) {
+ if (typeof define === 'function' && define.amd) {
+ // AMD. Make globaly available as well
+ define(['moment', 'jquery'], function (moment, jquery) {
+ return (root.daterangepicker = factory(moment, jquery));
+ });
+ } else if (typeof module === 'object' && module.exports) {
+ // Node / Browserify
+ //isomorphic issue
+ var jQuery = (typeof window != 'undefined') ? window.jQuery : undefined;
+ if (!jQuery) {
+ jQuery = require('jquery');
+ if (!jQuery.fn) jQuery.fn = {};
+ }
+ module.exports = factory(require('moment'), jQuery);
+ } else {
+ // Browser globals
+ root.daterangepicker = factory(root.moment, root.jQuery);
}
-
- factory(root, exports, momentjs, jQuery);
-
- // Finally, as a browser global.
- } else {
- root.daterangepicker = factory(root, {}, root.moment || moment, (root.jQuery || root.Zepto || root.ender || root.$));
- }
-
-}(this, function(root, daterangepicker, moment, $) {
-
+}(this, function(moment, $) {
var DateRangePicker = function(element, options, cb) {
//default settings for options
@@ -39,7 +33,6 @@
this.element = $(element);
this.startDate = moment().startOf('day');
this.endDate = moment().endOf('day');
- this.timeZone = moment().utcOffset();
this.minDate = false;
this.maxDate = false;
this.dateLimit = false;
@@ -47,12 +40,16 @@
this.singleDatePicker = false;
this.showDropdowns = false;
this.showWeekNumbers = false;
+ this.showISOWeekNumbers = false;
+ this.showCustomRangeLabel = true;
this.timePicker = false;
this.timePicker24Hour = false;
this.timePickerIncrement = 1;
this.timePickerSeconds = false;
+ this.linkedCalendars = true;
+ this.autoUpdateInput = true;
+ this.alwaysShowCalendars = false;
this.ranges = {};
- this.viewClass ="";
this.opens = 'right';
if (this.element.hasClass('pull-right'))
@@ -67,6 +64,7 @@
this.cancelClass = 'btn-default';
this.locale = {
+ direction: 'ltr',
format: 'MM/DD/YYYY',
separator: ' - ',
applyLabel: 'Apply',
@@ -75,7 +73,7 @@
customRangeLabel: 'Custom Range',
daysOfWeek: moment.weekdaysMin(),
monthNames: moment.monthsShort(),
- firstDay: moment.localeData()._week.dow
+ firstDay: moment.localeData().firstDayOfWeek()
};
this.callback = function() { };
@@ -94,13 +92,13 @@
options = $.extend(this.element.data(), options);
//html template for the picker UI
- if (typeof options.template !== 'string')
- options.template = '<div class="daterangepicker '+ options.viewClass+' dropdown-menu">' +
+ if (typeof options.template !== 'string' && !(options.template instanceof $))
+ options.template = '<div class="daterangepicker dropdown-menu">' +
'<div class="calendar left">' +
'<div class="daterangepicker_input">' +
- '<input class="input-mini" type="text" name="daterangepicker_start" value="" />' +
+ '<input class="input-mini form-control" type="text" name="daterangepicker_start" value="" />' +
'<i class="fa fa-calendar glyphicon glyphicon-calendar"></i>' +
- '<div class="calendar-time">' +
+ '<div class="calendar-time">' +
'<div></div>' +
'<i class="fa fa-clock-o glyphicon glyphicon-time"></i>' +
'</div>' +
@@ -109,9 +107,9 @@
'</div>' +
'<div class="calendar right">' +
'<div class="daterangepicker_input">' +
- '<input class="input-mini" type="text" name="daterangepicker_end" value="" />' +
+ '<input class="input-mini form-control" type="text" name="daterangepicker_end" value="" />' +
'<i class="fa fa-calendar glyphicon glyphicon-calendar"></i>' +
- '<div class="calendar-time">' +
+ '<div class="calendar-time">' +
'<div></div>' +
'<i class="fa fa-clock-o glyphicon glyphicon-time"></i>' +
'</div>' +
@@ -135,6 +133,9 @@
if (typeof options.locale === 'object') {
+ if (typeof options.locale.direction === 'string')
+ this.locale.direction = options.locale.direction;
+
if (typeof options.locale.format === 'string')
this.locale.format = options.locale.format;
@@ -163,6 +164,7 @@
this.locale.customRangeLabel = options.locale.customRangeLabel;
}
+ this.container.addClass(this.locale.direction);
if (typeof options.startDate === 'string')
this.startDate = moment(options.startDate, this.locale.format);
@@ -188,6 +190,14 @@
if (typeof options.maxDate === 'object')
this.maxDate = moment(options.maxDate);
+ // sanity check for bad options
+ if (this.minDate && this.startDate.isBefore(this.minDate))
+ this.startDate = this.minDate.clone();
+
+ // sanity check for bad options
+ if (this.maxDate && this.endDate.isAfter(this.maxDate))
+ this.endDate = this.maxDate.clone();
+
if (typeof options.applyClass === 'string')
this.applyClass = options.applyClass;
@@ -206,6 +216,9 @@
if (typeof options.showWeekNumbers === 'boolean')
this.showWeekNumbers = options.showWeekNumbers;
+ if (typeof options.showISOWeekNumbers === 'boolean')
+ this.showISOWeekNumbers = options.showISOWeekNumbers;
+
if (typeof options.buttonClasses === 'string')
this.buttonClasses = options.buttonClasses;
@@ -214,7 +227,10 @@
if (typeof options.showDropdowns === 'boolean')
this.showDropdowns = options.showDropdowns;
-
+
+ if (typeof options.showCustomRangeLabel === 'boolean')
+ this.showCustomRangeLabel = options.showCustomRangeLabel;
+
if (typeof options.singleDatePicker === 'boolean') {
this.singleDatePicker = options.singleDatePicker;
if (this.singleDatePicker)
@@ -236,6 +252,21 @@
if (typeof options.autoApply === 'boolean')
this.autoApply = options.autoApply;
+ if (typeof options.autoUpdateInput === 'boolean')
+ this.autoUpdateInput = options.autoUpdateInput;
+
+ if (typeof options.linkedCalendars === 'boolean')
+ this.linkedCalendars = options.linkedCalendars;
+
+ if (typeof options.isInvalidDate === 'function')
+ this.isInvalidDate = options.isInvalidDate;
+
+ if (typeof options.isCustomDate === 'function')
+ this.isCustomDate = options.isCustomDate;
+
+ if (typeof options.alwaysShowCalendars === 'boolean')
+ this.alwaysShowCalendars = options.alwaysShowCalendars;
+
// update day names order to firstDay
if (this.locale.firstDay != 0) {
var iterator = this.locale.firstDay;
@@ -269,19 +300,6 @@
}
}
- // bind the time zone used to build the calendar to either the timeZone passed in through the options or the zone of the startDate (which will be the local time zone by default)
- if (typeof options.timeZone === 'string' || typeof options.timeZone === 'number') {
- if (typeof options.timeZone === 'string' && typeof moment.tz !== 'undefined') {
- this.timeZone = moment.tz.zone(options.timeZone).parse(new Date) * -1; // Offset is positive if the timezone is behind UTC and negative if it is ahead.
- } else {
- this.timeZone = options.timeZone;
- }
- this.startDate.utcOffset(this.timeZone);
- this.endDate.utcOffset(this.timeZone);
- } else {
- this.timeZone = moment(this.startDate).utcOffset();
- }
-
if (typeof options.ranges === 'object') {
for (range in options.ranges) {
@@ -301,26 +319,33 @@
start = this.minDate.clone();
var maxDate = this.maxDate;
- if (this.dateLimit && start.clone().add(this.dateLimit).isAfter(maxDate))
+ if (this.dateLimit && maxDate && start.clone().add(this.dateLimit).isAfter(maxDate))
maxDate = start.clone().add(this.dateLimit);
if (maxDate && end.isAfter(maxDate))
end = maxDate.clone();
- // If the end of the range is before the minimum or the start of the range is
+ // If the end of the range is before the minimum or the start of the range is
// after the maximum, don't display this range option at all.
- if ((this.minDate && end.isBefore(this.minDate)) || (maxDate && start.isAfter(maxDate)))
+ if ((this.minDate && end.isBefore(this.minDate, this.timepicker ? 'minute' : 'day'))
+ || (maxDate && start.isAfter(maxDate, this.timepicker ? 'minute' : 'day')))
continue;
- this.ranges[range] = [start, end];
+ //Support unicode chars in the range names.
+ var elem = document.createElement('textarea');
+ elem.innerHTML = range;
+ var rangeHtml = elem.value;
+
+ this.ranges[rangeHtml] = [start, end];
}
var list = '<ul>';
for (range in this.ranges) {
- list += '<li>' + range + '</li>';
+ list += '<li data-range-key="' + range + '">' + range + '</li>';
+ }
+ if (this.showCustomRangeLabel) {
+ list += '<li data-range-key="' + this.locale.customRangeLabel + '">' + this.locale.customRangeLabel + '</li>';
}
- list += '<li>' + this.locale.customRangeLabel + '</li>';
list += '</ul>';
- this.container.find('.ranges ul').remove();
this.container.find('.ranges').prepend(list);
}
@@ -349,17 +374,24 @@
this.container.find('.calendar.left').addClass('single');
this.container.find('.calendar.left').show();
this.container.find('.calendar.right').hide();
- this.container.find('.daterangepicker_input input, .daterangepicker_input i').hide();
- if (!this.timePicker) {
+ this.container.find('.daterangepicker_input input, .daterangepicker_input > i').hide();
+ if (this.timePicker) {
+ this.container.find('.ranges ul').hide();
+ } else {
this.container.find('.ranges').hide();
}
}
- if (typeof options.ranges === 'undefined' && !this.singleDatePicker) {
+ if ((typeof options.ranges === 'undefined' && !this.singleDatePicker) || this.alwaysShowCalendars) {
this.container.addClass('show-calendar');
}
- this.container.removeClass('opensleft opensright').addClass('opens' + this.opens);
+ this.container.addClass('opens' + this.opens);
+
+ //swap the position of the predefined ranges if opens right
+ if (typeof options.ranges !== 'undefined' && this.opens == 'right') {
+ this.container.find('.ranges').prependTo( this.container.find('.calendar.left').parent() );
+ }
//apply CSS classes and labels to buttons
this.container.find('.applyBtn, .cancelBtn').addClass(this.buttonClasses);
@@ -377,28 +409,29 @@
this.container.find('.calendar')
.on('click.daterangepicker', '.prev', $.proxy(this.clickPrev, this))
.on('click.daterangepicker', '.next', $.proxy(this.clickNext, this))
- .on('click.daterangepicker', 'td.available', $.proxy(this.clickDate, this))
+ .on('mousedown.daterangepicker', 'td.available', $.proxy(this.clickDate, this))
.on('mouseenter.daterangepicker', 'td.available', $.proxy(this.hoverDate, this))
.on('mouseleave.daterangepicker', 'td.available', $.proxy(this.updateFormInputs, this))
.on('change.daterangepicker', 'select.yearselect', $.proxy(this.monthOrYearChanged, this))
.on('change.daterangepicker', 'select.monthselect', $.proxy(this.monthOrYearChanged, this))
.on('change.daterangepicker', 'select.hourselect,select.minuteselect,select.secondselect,select.ampmselect', $.proxy(this.timeChanged, this))
.on('click.daterangepicker', '.daterangepicker_input input', $.proxy(this.showCalendars, this))
- .on('change.daterangepicker', '.daterangepicker_input input', $.proxy(this.formInputsChanged, this))
- .on('change.daterangepicker', '.daterangepicker_input input', $.proxy(this.updateFormInputs, this));
+ .on('focus.daterangepicker', '.daterangepicker_input input', $.proxy(this.formInputsFocused, this))
+ .on('blur.daterangepicker', '.daterangepicker_input input', $.proxy(this.formInputsBlurred, this))
+ .on('change.daterangepicker', '.daterangepicker_input input', $.proxy(this.formInputsChanged, this));
this.container.find('.ranges')
.on('click.daterangepicker', 'button.applyBtn', $.proxy(this.clickApply, this))
.on('click.daterangepicker', 'button.cancelBtn', $.proxy(this.clickCancel, this))
.on('click.daterangepicker', 'li', $.proxy(this.clickRange, this))
- /* .on('mouseenter.daterangepicker', 'li', $.proxy(this.hoverRange, this))
- .on('mouseleave.daterangepicker', 'li', $.proxy(this.updateFormInputs, this));*/
+ .on('mouseenter.daterangepicker', 'li', $.proxy(this.hoverRange, this))
+ .on('mouseleave.daterangepicker', 'li', $.proxy(this.updateFormInputs, this));
- if (this.element.is('input')) {
+ if (this.element.is('input') || this.element.is('button')) {
this.element.on({
'click.daterangepicker': $.proxy(this.show, this),
'focus.daterangepicker': $.proxy(this.show, this),
- 'keyup.daterangepicker': $.proxy(this.controlChanged, this),
+ 'keyup.daterangepicker': $.proxy(this.elementChanged, this),
'keydown.daterangepicker': $.proxy(this.keydown, this)
});
} else {
@@ -409,10 +442,10 @@
// if attached to a text input, set the initial value
//
- if (this.element.is('input') && !this.singleDatePicker) {
+ if (this.element.is('input') && !this.singleDatePicker && this.autoUpdateInput) {
this.element.val(this.startDate.format(this.locale.format) + this.locale.separator + this.endDate.format(this.locale.format));
this.element.trigger('change');
- } else if (this.element.is('input')) {
+ } else if (this.element.is('input') && this.autoUpdateInput) {
this.element.val(this.startDate.format(this.locale.format));
this.element.trigger('change');
}
@@ -425,7 +458,7 @@
setStartDate: function(startDate) {
if (typeof startDate === 'string')
- this.startDate = moment(startDate, this.locale.format).utcOffset(this.timeZone);
+ this.startDate = moment(startDate, this.locale.format);
if (typeof startDate === 'object')
this.startDate = moment(startDate);
@@ -436,18 +469,27 @@
if (this.timePicker && this.timePickerIncrement)
this.startDate.minute(Math.round(this.startDate.minute() / this.timePickerIncrement) * this.timePickerIncrement);
- if (this.minDate && this.startDate.isBefore(this.minDate))
+ if (this.minDate && this.startDate.isBefore(this.minDate)) {
this.startDate = this.minDate;
+ if (this.timePicker && this.timePickerIncrement)
+ this.startDate.minute(Math.round(this.startDate.minute() / this.timePickerIncrement) * this.timePickerIncrement);
+ }
- if (this.maxDate && this.startDate.isAfter(this.maxDate))
+ if (this.maxDate && this.startDate.isAfter(this.maxDate)) {
this.startDate = this.maxDate;
+ if (this.timePicker && this.timePickerIncrement)
+ this.startDate.minute(Math.floor(this.startDate.minute() / this.timePickerIncrement) * this.timePickerIncrement);
+ }
+
+ if (!this.isShowing)
+ this.updateElement();
this.updateMonthsInView();
},
setEndDate: function(endDate) {
if (typeof endDate === 'string')
- this.endDate = moment(endDate, this.locale.format).utcOffset(this.timeZone);
+ this.endDate = moment(endDate, this.locale.format);
if (typeof endDate === 'object')
this.endDate = moment(endDate);
@@ -467,9 +509,22 @@
if (this.dateLimit && this.startDate.clone().add(this.dateLimit).isBefore(this.endDate))
this.endDate = this.startDate.clone().add(this.dateLimit);
+ this.previousRightTime = this.endDate.clone();
+
+ if (!this.isShowing)
+ this.updateElement();
+
this.updateMonthsInView();
},
+ isInvalidDate: function() {
+ return false;
+ },
+
+ isCustomDate: function() {
+ return false;
+ },
+
updateView: function() {
if (this.timePicker) {
this.renderTimePicker('left');
@@ -480,6 +535,13 @@
this.container.find('.right .calendar-time select').removeAttr('disabled').removeClass('disabled');
}
}
+ if (this.endDate) {
+ this.container.find('input[name="daterangepicker_end"]').removeClass('active');
+ this.container.find('input[name="daterangepicker_start"]').addClass('active');
+ } else {
+ this.container.find('input[name="daterangepicker_end"]').addClass('active');
+ this.container.find('input[name="daterangepicker_start"]').removeClass('active');
+ }
this.updateMonthsInView();
this.updateCalendars();
this.updateFormInputs();
@@ -487,19 +549,33 @@
updateMonthsInView: function() {
if (this.endDate) {
- this.leftCalendar.month = this.startDate.clone().day(1);
- if (this.endDate.month() != this.startDate.month() || this.endDate.year() != this.startDate.year()) {
- this.rightCalendar.month = this.endDate.clone().day(1);
+
+ //if both dates are visible already, do nothing
+ if (!this.singleDatePicker && this.leftCalendar.month && this.rightCalendar.month &&
+ (this.startDate.format('YYYY-MM') == this.leftCalendar.month.format('YYYY-MM') || this.startDate.format('YYYY-MM') == this.rightCalendar.month.format('YYYY-MM'))
+ &&
+ (this.endDate.format('YYYY-MM') == this.leftCalendar.month.format('YYYY-MM') || this.endDate.format('YYYY-MM') == this.rightCalendar.month.format('YYYY-MM'))
+ ) {
+ return;
+ }
+
+ this.leftCalendar.month = this.startDate.clone().date(2);
+ if (!this.linkedCalendars && (this.endDate.month() != this.startDate.month() || this.endDate.year() != this.startDate.year())) {
+ this.rightCalendar.month = this.endDate.clone().date(2);
} else {
- this.rightCalendar.month = this.startDate.clone().day(1).add(1, 'month');
+ this.rightCalendar.month = this.startDate.clone().date(2).add(1, 'month');
}
+
} else {
if (this.leftCalendar.month.format('YYYY-MM') != this.startDate.format('YYYY-MM') && this.rightCalendar.month.format('YYYY-MM') != this.startDate.format('YYYY-MM')) {
- console.log('here2');
- this.leftCalendar.month = this.startDate.clone().day(1);
- this.rightCalendar.month = this.startDate.clone().day(1).add(1, 'month');
+ this.leftCalendar.month = this.startDate.clone().date(2);
+ this.rightCalendar.month = this.startDate.clone().date(2).add(1, 'month');
}
}
+ if (this.maxDate && this.linkedCalendars && !this.singleDatePicker && this.rightCalendar.month > this.maxDate) {
+ this.rightCalendar.month = this.maxDate.clone().date(2);
+ this.leftCalendar.month = this.maxDate.clone().date(2).subtract(1, 'month');
+ }
},
updateCalendars: function() {
@@ -522,7 +598,7 @@
minute = parseInt(this.container.find('.right .minuteselect').val(), 10);
second = this.timePickerSeconds ? parseInt(this.container.find('.right .secondselect').val(), 10) : 0;
if (!this.timePicker24Hour) {
- var ampm = this.container.find('.left .ampmselect').val();
+ var ampm = this.container.find('.right .ampmselect').val();
if (ampm === 'PM' && hour < 12)
hour += 12;
if (ampm === 'AM' && hour === 12)
@@ -540,28 +616,7 @@
this.container.find('.ranges li').removeClass('active');
if (this.endDate == null) return;
- var customRange = true;
- var i = 0;
- for (var range in this.ranges) {
- if (this.timePicker) {
- if (this.startDate.isSame(this.ranges[range][0]) && this.endDate.isSame(this.ranges[range][1])) {
- customRange = false;
- this.chosenLabel = this.container.find('.ranges li:eq(' + i + ')').addClass('active').html();
- }
- } else {
- //ignore times when comparing dates if time picker is not enabled
- if (this.startDate.format('YYYY-MM-DD') == this.ranges[range][0].format('YYYY-MM-DD') && this.endDate.format('YYYY-MM-DD') == this.ranges[range][1].format('YYYY-MM-DD')) {
- customRange = false;
- this.chosenLabel = this.container.find('.ranges li:eq(' + i + ')').addClass('active').html();
- }
- }
- i++;
- }
- if (customRange) {
- this.chosenLabel = this.container.find('.ranges li:last').addClass('active').html();
- this.showCalendars();
- }
-
+ this.calculateChosenLabel();
},
renderCalendar: function(side) {
@@ -601,8 +656,7 @@
if (dayOfWeek == this.locale.firstDay)
startDay = daysInLastMonth - 6;
- // Possible patch for issue #626 https://github.com/dangrossman/bootstrap-daterangepicker/issues/626
- var curDate = moment([lastYear, lastMonth, startDay, 12, minute, second]); // .utcOffset(this.timeZone);
+ var curDate = moment([lastYear, lastMonth, startDay, 12, minute, second]);
var col, row;
for (var i = 0, col = 0, row = 0; i < 42; i++, col++, curDate = moment(curDate).add(24, 'hour')) {
@@ -637,17 +691,18 @@
var minDate = side == 'left' ? this.minDate : this.startDate;
var maxDate = this.maxDate;
var selected = side == 'left' ? this.startDate : this.endDate;
+ var arrow = this.locale.direction == 'ltr' ? {left: 'chevron-left', right: 'chevron-right'} : {left: 'chevron-right', right: 'chevron-left'};
- html = '<table class="table-condensed">';
+ var html = '<table class="table-condensed">';
html += '<thead>';
html += '<tr>';
// add empty cell for week number
- if (this.showWeekNumbers)
+ if (this.showWeekNumbers || this.showISOWeekNumbers)
html += '<th></th>';
- if (!minDate || minDate.isBefore(calendar.firstDay)) {
- html += '<th class="prev available"><i class="fa fa-chevron-left glyphicon glyphicon-chevron-left"></i></th>';
+ if ((!minDate || minDate.isBefore(calendar.firstDay)) && (!this.linkedCalendars || side == 'left')) {
+ html += '<th class="prev available"><i class="fa fa-' + arrow.left + ' glyphicon glyphicon-' + arrow.left + '"></i></th>';
} else {
html += '<th></th>';
}
@@ -668,6 +723,10 @@
monthHtml += "<option value='" + m + "'" +
(m === currentMonth ? " selected='selected'" : "") +
">" + this.locale.monthNames[m] + "</option>";
+ } else {
+ monthHtml += "<option value='" + m + "'" +
+ (m === currentMonth ? " selected='selected'" : "") +
+ " disabled='disabled'>" + this.locale.monthNames[m] + "</option>";
}
}
monthHtml += "</select>";
@@ -684,8 +743,8 @@
}
html += '<th colspan="5" class="month">' + dateHtml + '</th>';
- if (!maxDate || maxDate.isAfter(calendar.lastDay)) {
- html += '<th class="next available"><i class="fa fa-chevron-right glyphicon glyphicon-chevron-right"></i></th>';
+ if ((!maxDate || maxDate.isAfter(calendar.lastDay)) && (!this.linkedCalendars || side == 'right' || this.singleDatePicker)) {
+ html += '<th class="next available"><i class="fa fa-' + arrow.right + ' glyphicon glyphicon-' + arrow.right + '"></i></th>';
} else {
html += '<th></th>';
}
@@ -694,7 +753,7 @@
html += '<tr>';
// add week number label
- if (this.showWeekNumbers)
+ if (this.showWeekNumbers || this.showISOWeekNumbers)
html += '<th class="week">' + this.locale.weekLabel + '</th>';
$.each(this.locale.daysOfWeek, function(index, dayOfWeek) {
@@ -720,11 +779,13 @@
// add week number
if (this.showWeekNumbers)
html += '<td class="week">' + calendar[row][0].week() + '</td>';
+ else if (this.showISOWeekNumbers)
+ html += '<td class="week">' + calendar[row][0].isoWeek() + '</td>';
for (var col = 0; col < 7; col++) {
var classes = [];
-
+
//highlight today's date
if (calendar[row][col].isSame(new Date(), "day"))
classes.push('today');
@@ -745,6 +806,10 @@
if (maxDate && calendar[row][col].isAfter(maxDate, 'day'))
classes.push('off', 'disabled');
+ //don't allow selection of date if a custom function decides it's invalid
+ if (this.isInvalidDate(calendar[row][col]))
+ classes.push('off', 'disabled');
+
//highlight the currently selected start date
if (calendar[row][col].format('YYYY-MM-DD') == this.startDate.format('YYYY-MM-DD'))
classes.push('active', 'start-date');
@@ -757,8 +822,17 @@
if (this.endDate != null && calendar[row][col] > this.startDate && calendar[row][col] < this.endDate)
classes.push('in-range');
+ //apply custom classes for this date
+ var isCustom = this.isCustomDate(calendar[row][col]);
+ if (isCustom !== false) {
+ if (typeof isCustom === 'string')
+ classes.push(isCustom);
+ else
+ Array.prototype.push.apply(classes, isCustom);
+ }
+
var cname = '', disabled = false;
- for (var i in classes) {
+ for (var i = 0; i < classes.length; i++) {
cname += classes[i] + ' ';
if (classes[i] == 'disabled')
disabled = true;
@@ -766,7 +840,7 @@
if (!disabled)
cname += 'available';
- html += '<td class="' + cname.trim() + '" data-title="' + 'r' + row + 'c' + col + '">' + calendar[row][col].date() + '</td>';
+ html += '<td class="' + cname.replace(/^\s+|\s+$/g, '') + '" data-title="' + 'r' + row + 'c' + col + '">' + calendar[row][col].date() + '</td>';
}
html += '</tr>';
@@ -780,8 +854,12 @@
},
renderTimePicker: function(side) {
-
- var selected, minDate, maxDate = this.maxDate;
+
+ // Don't bother updating the time picker if it's currently disabled
+ // because an end date hasn't been clicked yet
+ if (side == 'right' && !this.endDate) return;
+
+ var html, selected, minDate, maxDate = this.maxDate;
if (this.dateLimit && (!this.maxDate || this.startDate.clone().add(this.dateLimit).isAfter(this.maxDate)))
maxDate = this.startDate.clone().add(this.dateLimit);
@@ -790,10 +868,35 @@
selected = this.startDate.clone();
minDate = this.minDate;
} else if (side == 'right') {
- selected = this.endDate ? this.endDate.clone() : this.startDate.clone();
+ selected = this.endDate.clone();
minDate = this.startDate;
+
+ //Preserve the time already selected
+ var timeSelector = this.container.find('.calendar.right .calendar-time div');
+ if (!this.endDate && timeSelector.html() != '') {
+
+ selected.hour(timeSelector.find('.hourselect option:selected').val() || selected.hour());
+ selected.minute(timeSelector.find('.minuteselect option:selected').val() || selected.minute());
+ selected.second(timeSelector.find('.secondselect option:selected').val() || selected.second());
+
+ if (!this.timePicker24Hour) {
+ var ampm = timeSelector.find('.ampmselect option:selected').val();
+ if (ampm === 'PM' && selected.hour() < 12)
+ selected.hour(selected.hour() + 12);
+ if (ampm === 'AM' && selected.hour() === 12)
+ selected.hour(0);
+ }
+
+ }
+
+ if (selected.isBefore(this.startDate))
+ selected = this.startDate.clone();
+
+ if (maxDate && selected.isAfter(maxDate))
+ selected = maxDate.clone();
+
}
-
+
//
// hours
//
@@ -912,6 +1015,11 @@
},
updateFormInputs: function() {
+
+ //ignore mouse movements while an above-calendar text input has focus
+ if (this.container.find('input[name=daterangepicker_start]').is(":focus") || this.container.find('input[name=daterangepicker_end]').is(":focus"))
+ return;
+
this.container.find('input[name=daterangepicker_start]').val(this.startDate.format(this.locale.format));
if (this.endDate)
this.container.find('input[name=daterangepicker_end]').val(this.endDate.format(this.locale.format));
@@ -921,6 +1029,7 @@
} else {
this.container.find('button.applyBtn').attr('disabled', 'disabled');
}
+
},
move: function() {
@@ -934,7 +1043,7 @@
};
parentRightEdge = this.parentEl[0].clientWidth + this.parentEl.offset().left;
}
-
+
if (this.drops == 'up')
containerTop = this.element.offset().top - this.container.outerHeight() - parentOffset.top;
else
@@ -986,6 +1095,7 @@
// Create a click proxy that is private to this instance of datepicker, for unbinding
this._outsideClickProxy = $.proxy(function(e) { this.outsideClick(e); }, this);
+
// Bind global datepicker mousedown for hiding and
$(document)
.on('mousedown.daterangepicker', this._outsideClickProxy)
@@ -996,9 +1106,13 @@
// and also close when focus changes to outside the picker (eg. tabbing between controls)
.on('focusin.daterangepicker', this._outsideClickProxy);
+ // Reposition the picker if the window is resized while it's open
+ $(window).on('resize.daterangepicker', $.proxy(function(e) { this.move(e); }, this));
+
this.oldStartDate = this.startDate.clone();
this.oldEndDate = this.endDate.clone();
-
+ this.previousRightTime = this.endDate.clone();
+
this.updateView();
this.container.show();
this.move();
@@ -1020,15 +1134,10 @@
this.callback(this.startDate, this.endDate, this.chosenLabel);
//if picker is attached to a text input, update it
- if (this.element.is('input') && !this.singleDatePicker) {
- this.element.val(this.startDate.format(this.locale.format) + this.locale.separator + this.endDate.format(this.locale.format));
- this.element.trigger('change');
- } else if (this.element.is('input')) {
- this.element.val(this.startDate.format(this.locale.format));
- this.element.trigger('change');
- }
+ this.updateElement();
$(document).off('.daterangepicker');
+ $(window).off('.daterangepicker');
this.container.hide();
this.element.trigger('hide.daterangepicker', this);
this.isShowing = false;
@@ -1054,6 +1163,7 @@
target.closest('.calendar-table').length
) return;
this.hide();
+ this.element.trigger('outsideClick.daterangepicker', this);
},
showCalendars: function() {
@@ -1068,7 +1178,13 @@
},
hoverRange: function(e) {
- var label = e.target.innerHTML;
+
+ //ignore mouse movements while an above-calendar text input has focus
+ if (this.container.find('input[name=daterangepicker_start]').is(":focus") || this.container.find('input[name=daterangepicker_end]').is(":focus"))
+ return;
+
+ var label = e.target.getAttribute('data-range-key');
+
if (label == this.locale.customRangeLabel) {
this.updateView();
} else {
@@ -1076,10 +1192,11 @@
this.container.find('input[name=daterangepicker_start]').val(dates[0].format(this.locale.format));
this.container.find('input[name=daterangepicker_end]').val(dates[1].format(this.locale.format));
}
+
},
clickRange: function(e) {
- var label = e.target.innerHTML;
+ var label = e.target.getAttribute('data-range-key');
this.chosenLabel = label;
if (label == this.locale.customRangeLabel) {
this.showCalendars();
@@ -1093,8 +1210,9 @@
this.endDate.endOf('day');
}
- this.hideCalendars();
- this.hide();
+ if (!this.alwaysShowCalendars)
+ this.hideCalendars();
+ this.clickApply();
}
},
@@ -1102,6 +1220,8 @@
var cal = $(e.target).parents('.calendar');
if (cal.hasClass('left')) {
this.leftCalendar.month.subtract(1, 'month');
+ if (this.linkedCalendars)
+ this.rightCalendar.month.subtract(1, 'month');
} else {
this.rightCalendar.month.subtract(1, 'month');
}
@@ -1114,27 +1234,59 @@
this.leftCalendar.month.add(1, 'month');
} else {
this.rightCalendar.month.add(1, 'month');
+ if (this.linkedCalendars)
+ this.leftCalendar.month.add(1, 'month');
}
this.updateCalendars();
},
hoverDate: function(e) {
+ //ignore mouse movements while an above-calendar text input has focus
+ //if (this.container.find('input[name=daterangepicker_start]').is(":focus") || this.container.find('input[name=daterangepicker_end]').is(":focus"))
+ // return;
+
//ignore dates that can't be selected
if (!$(e.target).hasClass('available')) return;
+ //have the text inputs above calendars reflect the date being hovered over
var title = $(e.target).attr('data-title');
var row = title.substr(1, 1);
var col = title.substr(3, 1);
var cal = $(e.target).parents('.calendar');
var date = cal.hasClass('left') ? this.leftCalendar.calendar[row][col] : this.rightCalendar.calendar[row][col];
- if (this.endDate) {
+ if (this.endDate && !this.container.find('input[name=daterangepicker_start]').is(":focus")) {
this.container.find('input[name=daterangepicker_start]').val(date.format(this.locale.format));
- } else {
+ } else if (!this.endDate && !this.container.find('input[name=daterangepicker_end]').is(":focus")) {
this.container.find('input[name=daterangepicker_end]').val(date.format(this.locale.format));
}
+ //highlight the dates between the start date and the date being hovered as a potential end date
+ var leftCalendar = this.leftCalendar;
+ var rightCalendar = this.rightCalendar;
+ var startDate = this.startDate;
+ if (!this.endDate) {
+ this.container.find('.calendar td').each(function(index, el) {
+
+ //skip week numbers, only look at dates
+ if ($(el).hasClass('week')) return;
+
+ var title = $(el).attr('data-title');
+ var row = title.substr(1, 1);
+ var col = title.substr(3, 1);
+ var cal = $(el).parents('.calendar');
+ var dt = cal.hasClass('left') ? leftCalendar.calendar[row][col] : rightCalendar.calendar[row][col];
+
+ if ((dt.isAfter(startDate) && dt.isBefore(date)) || dt.isSame(date, 'day')) {
+ $(el).addClass('in-range');
+ } else {
+ $(el).removeClass('in-range');
+ }
+
+ });
+ }
+
},
clickDate: function(e) {
@@ -1153,13 +1305,14 @@
// * if the time picker is enabled, apply the hour/minute/second from the select boxes to the clicked date
// * if autoapply is enabled, and an end date was chosen, apply the selection
// * if single date picker mode, and time picker isn't enabled, apply the selection immediately
+ // * if one of the inputs above the calendars was focused, cancel that manual input
//
- if (this.endDate || date.isBefore(this.startDate)) {
+ if (this.endDate || date.isBefore(this.startDate, 'day')) { //picking start
if (this.timePicker) {
var hour = parseInt(this.container.find('.left .hourselect').val(), 10);
if (!this.timePicker24Hour) {
- var ampm = cal.find('.ampmselect').val();
+ var ampm = this.container.find('.left .ampmselect').val();
if (ampm === 'PM' && hour < 12)
hour += 12;
if (ampm === 'AM' && hour === 12)
@@ -1171,7 +1324,11 @@
}
this.endDate = null;
this.setStartDate(date.clone());
- } else {
+ } else if (!this.endDate && date.isBefore(this.startDate)) {
+ //special case: clicking the same date for start/end,
+ //but the time of the end date is before the start date
+ this.setEndDate(this.startDate.clone());
+ } else { // picking end
if (this.timePicker) {
var hour = parseInt(this.container.find('.right .hourselect').val(), 10);
if (!this.timePicker24Hour) {
@@ -1186,8 +1343,10 @@
date = date.clone().hour(hour).minute(minute).second(second);
}
this.setEndDate(date.clone());
- if (this.autoApply)
- this.clickApply();
+ if (this.autoApply) {
+ this.calculateChosenLabel();
+ this.clickApply();
+ }
}
if (this.singleDatePicker) {
@@ -1198,6 +1357,35 @@
this.updateView();
+ //This is to cancel the blur event handler if the mouse was in one of the inputs
+ e.stopPropagation();
+
+ },
+
+ calculateChosenLabel: function() {
+ var customRange = true;
+ var i = 0;
+ for (var range in this.ranges) {
+ if (this.timePicker) {
+ if (this.startDate.isSame(this.ranges[range][0]) && this.endDate.isSame(this.ranges[range][1])) {
+ customRange = false;
+ this.chosenLabel = this.container.find('.ranges li:eq(' + i + ')').addClass('active').html();
+ break;
+ }
+ } else {
+ //ignore times when comparing dates if time picker is not enabled
+ if (this.startDate.format('YYYY-MM-DD') == this.ranges[range][0].format('YYYY-MM-DD') && this.endDate.format('YYYY-MM-DD') == this.ranges[range][1].format('YYYY-MM-DD')) {
+ customRange = false;
+ this.chosenLabel = this.container.find('.ranges li:eq(' + i + ')').addClass('active').html();
+ break;
+ }
+ }
+ i++;
+ }
+ if (customRange && this.showCustomRangeLabel) {
+ this.chosenLabel = this.container.find('.ranges li:last').addClass('active').html();
+ this.showCalendars();
+ }
},
clickApply: function(e) {
@@ -1242,7 +1430,15 @@
}
}
- this[leftOrRight+'Calendar'].month.month(month).year(year);
+ if (isLeft) {
+ this.leftCalendar.month.month(month).year(year);
+ if (this.linkedCalendars)
+ this.rightCalendar.month = this.leftCalendar.month.clone().add(1, 'month');
+ } else {
+ this.rightCalendar.month.month(month).year(year);
+ if (this.linkedCalendars)
+ this.leftCalendar.month = this.rightCalendar.month.clone().subtract(1, 'month');
+ }
this.updateCalendars();
},
@@ -1269,8 +1465,11 @@
start.minute(minute);
start.second(second);
this.setStartDate(start);
- if (this.singleDatePicker)
+ if (this.singleDatePicker) {
this.endDate = this.startDate.clone();
+ } else if (this.endDate && this.endDate.format('YYYY-MM-DD') == start.format('YYYY-MM-DD') && this.endDate.isBefore(start)) {
+ this.setEndDate(start.clone());
+ }
} else if (this.endDate) {
var end = this.endDate.clone();
end.hour(hour);
@@ -1312,36 +1511,73 @@
}
- this.updateCalendars();
- if (this.timePicker) {
- this.renderTimePicker('left');
- this.renderTimePicker('right');
+ this.updateView();
+ },
+
+ formInputsFocused: function(e) {
+
+ // Highlight the focused input
+ this.container.find('input[name="daterangepicker_start"], input[name="daterangepicker_end"]').removeClass('active');
+ $(e.target).addClass('active');
+
+ // Set the state such that if the user goes back to using a mouse,
+ // the calendars are aware we're selecting the end of the range, not
+ // the start. This allows someone to edit the end of a date range without
+ // re-selecting the beginning, by clicking on the end date input then
+ // using the calendar.
+ var isRight = $(e.target).closest('.calendar').hasClass('right');
+ if (isRight) {
+ this.endDate = null;
+ this.setStartDate(this.startDate.clone());
+ this.updateView();
}
+
},
- controlChanged: function() {
+ formInputsBlurred: function(e) {
+
+ // this function has one purpose right now: if you tab from the first
+ // text input to the second in the UI, the endDate is nulled so that
+ // you can click another, but if you tab out without clicking anything
+ // or changing the input value, the old endDate should be retained
+
+ if (!this.endDate) {
+ var val = this.container.find('input[name="daterangepicker_end"]').val();
+ var end = moment(val, this.locale.format);
+ if (end.isValid()) {
+ this.setEndDate(end);
+ this.updateView();
+ }
+ }
+
+ },
+
+ elementChanged: function() {
if (!this.element.is('input')) return;
if (!this.element.val().length) return;
+ if (this.element.val().length < this.locale.format.length) return;
var dateString = this.element.val().split(this.locale.separator),
start = null,
end = null;
if (dateString.length === 2) {
- start = moment(dateString[0], this.locale.format).utcOffset(this.timeZone);
- end = moment(dateString[1], this.locale.format).utcOffset(this.timeZone);
+ start = moment(dateString[0], this.locale.format);
+ end = moment(dateString[1], this.locale.format);
}
if (this.singleDatePicker || start === null || end === null) {
- start = moment(this.element.val(), this.locale.format).utcOffset(this.timeZone);
+ start = moment(this.element.val(), this.locale.format);
end = start;
}
+ if (!start.isValid() || !end.isValid()) return;
+
this.setStartDate(start);
this.setEndDate(end);
this.updateView();
},
-
+
keydown: function(e) {
//hide on tab or enter
if ((e.keyCode === 9) || (e.keyCode === 13)) {
@@ -1349,6 +1585,16 @@
}
},
+ updateElement: function() {
+ if (this.element.is('input') && !this.singleDatePicker && this.autoUpdateInput) {
+ this.element.val(this.startDate.format(this.locale.format) + this.locale.separator + this.endDate.format(this.locale.format));
+ this.element.trigger('change');
+ } else if (this.element.is('input') && this.autoUpdateInput) {
+ this.element.val(this.startDate.format(this.locale.format));
+ this.element.trigger('change');
+ }
+ },
+
remove: function() {
this.container.remove();
this.element.off('.daterangepicker');
@@ -1367,4 +1613,6 @@
return this;
};
+ return DateRangePicker;
+
}));
http://git-wip-us.apache.org/repos/asf/ambari/blob/74b1952f/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/DatePickerLayout.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/DatePickerLayout.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/DatePickerLayout.js
index 1d7eccf..3beb776 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/DatePickerLayout.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/DatePickerLayout.js
@@ -59,6 +59,7 @@ define(['require',
this.bindEvents();
this.graphParams = {};
this.unit = this.params.unit ? this.params.unit : "+1HOUR";
+ this.isEventTriggerdFromVent = false;
},
bindEvents: function() {
this.listenTo(this.vent, "tab:refresh", function(params) {
@@ -68,6 +69,7 @@ define(['require',
this.setValues(options);
}, this);
this.listenTo(this.vent, "date:click", function(options) {
+ this.isEventTriggerdFromVent = true;
this.setValues(options);
this.ui.dateRange.data('daterangepicker').clickApply();
}, this);
@@ -165,6 +167,15 @@ define(['require',
this.ui.dateRange.on('apply.daterangepicker ', function(ev, picker) {
+ if(! that.isEventTriggerdFromVent && !(_.isUndefined(picker.chosenLabel)) ){
+ that.dateRangeLabel = picker.chosenLabel;
+ }else{
+ that.isEventTriggerdFromVent = false;
+ }
+ if (that.dateRangeLabel !== "Custom Range") {
+ var last1Hour = that.dateUtil.getRelativeDateFromString(that.dateRangeLabel);
+ that.setDateText(last1Hour[0], last1Hour[1]);
+ }
that.ui.dateRangeTitle.html(that.dateRangeLabel);
that.unit = that.checkDateRange(picker);
var options = {
@@ -178,7 +189,8 @@ define(['require',
});
this.ui.dateRange.on('show.daterangepicker', function(ev, picker) {
elem.find('li').removeClass('active');
- elem.find('li:contains(' + that.dateRangeLabel + ')').addClass('active')
+ elem.find('li:contains(' + that.dateRangeLabel + ')').addClass('active');
+ picker.chosenLabel = that.dateRangeLabel;
});
this.ui.dateRange.on('hide.daterangepicker', function(ev, picker) {
that.pickerOpend = true
@@ -196,15 +208,6 @@ define(['require',
that.ui.dateRange.data('daterangepicker').clickApply();
});
-
- elem.on("click", '.ranges ul li', function(e) {
- that.dateRangeLabel = $(this).text();
- if (that.dateRangeLabel !== "Custom Range") {
- var last1Hour = that.dateUtil.getRelativeDateFromString(that.dateRangeLabel);
- that.setDateText(last1Hour[0], last1Hour[1]);
- that.ui.dateRange.data('daterangepicker').clickApply();
- }
- });
},
checkDateRange: function(picker) {
return this.dateUtil.calculateUnit(picker)
http://git-wip-us.apache.org/repos/asf/ambari/blob/74b1952f/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
index e2b2457..0085f06 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
@@ -205,7 +205,8 @@ define(['require',
parentEl: that.$el,
fetch: true,
rangeLabel: true,
- width: '55%'
+ datePickerPosition : "left",
+ width: '65%'
}));
});
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/74b1952f/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
index 3ed84b3..c1655f0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
@@ -275,7 +275,7 @@ define(['backbone',
hideFireButton : false,
buttonLabel : "Apply",
parentEl: that.$el.find(".row").first(),
- datePickerPosition : "right"
+ datePickerPosition : "left"
}));
});
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/74b1952f/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
index 45e9144..240cc4d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
@@ -2910,6 +2910,7 @@ td.filterName {
content: "";
display: table;
}
+
table.backgrid{
overflow: auto !important;
}
@@ -2926,12 +2927,6 @@ button.defaultCancelBtn{
color: #333333 !important;
background-color: #ffffff;
}
-ul.user-profile{
- list-style: none;
- margin: 0;
- -webkit-padding-start: 0;
-}
-ul.user-profile i{
- font-size: 26px;
- margin-right: 20px;
+.comparisonTab .daterangepicker .ranges{
+ float: right !important;
}
\ No newline at end of file
[11/55] [abbrv] ambari git commit: AMBARI-18194. Hsi-Hs2: Mondrian
tests intermittently fail with "java.io.IOException: Connection reset by
peer" (aonishuk)
Posted by ol...@apache.org.
AMBARI-18194. Hsi-Hs2: Mondrian tests intermittently fail with "java.io.IOException: Connection reset by peer" (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2bc8690a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2bc8690a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2bc8690a
Branch: refs/heads/branch-dev-logsearch
Commit: 2bc8690a33f48b61def8278dd8abba938456e5f2
Parents: c758b7f
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Aug 18 17:24:49 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Aug 18 17:24:49 2016 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py | 1 +
.../stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml | 6 ++++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc8690a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
index 33006e1..5544085 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
@@ -98,6 +98,7 @@ mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
# upgrades would cause these directories to have a version instead of "current"
# which would cause a lot of problems when writing out hadoop-env.sh; instead
# force the use of "current" in the hook
+hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000")
hadoop_home = stack_select.get_hadoop_dir("home", force_latest_on_upgrade=True)
hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc8690a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
index 89c26a5..24e0193 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
@@ -147,13 +147,15 @@ export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}
export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS"
-{% if is_datanode_max_locked_memory_set %}
+
# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin.
# Makes sense to fix only when runing DN as root
if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
+ {% if is_datanode_max_locked_memory_set %}
ulimit -l {{datanode_max_locked_memory}}
+ {% endif %}
+ ulimit -n {{hdfs_user_nofile_limit}}
fi
-{% endif %}
</value>
<value-attributes>
<type>content</type>
[54/55] [abbrv] ambari git commit: AMBARI-18161. Support UNKNOWN log
level for Log Search (Dharmesh Makwana via oleewere)
Posted by ol...@apache.org.
AMBARI-18161. Support UNKNOWN log level for Log Search (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d8bb21c2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d8bb21c2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d8bb21c2
Branch: refs/heads/branch-dev-logsearch
Commit: d8bb21c2b069326fead81c2a509e4111987d1db0
Parents: a971c1e
Author: oleewere <ol...@gmail.com>
Authored: Mon Aug 22 11:01:36 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:04 2016 +0200
----------------------------------------------------------------------
.../org/apache/ambari/logfeeder/OutputMgr.java | 7 +-
.../logfeeder/logconfig/LogFeederConstants.java | 1 +
.../logsearch/common/LogSearchConstants.java | 3 +
.../logsearch/graph/GraphDataGenerator.java | 18 +----
.../ambari/logsearch/manager/LogsMgr.java | 84 +++-----------------
.../ambari/logsearch/util/ConfigUtil.java | 14 ----
.../main/webapp/scripts/views/common/Header.js | 2 +-
.../scripts/views/dashboard/LogLevelBoxView.js | 3 +-
.../scripts/views/dashboard/LogLevelView.js | 1 +
.../views/filter/CreateLogfeederFilterView.js | 2 +-
.../scripts/views/graphs/GraphLayoutView.js | 3 +-
.../scripts/views/graphs/GridGraphLayoutView.js | 3 +-
.../webapp/scripts/views/tabs/LogFileView.js | 2 +-
.../troubleshoot/TroubleShootLayoutView.js | 3 +-
.../src/main/webapp/styles/style.css | 46 +++++++++--
.../dashboard/LogLevelBoxView_tmpl.html | 17 ++--
.../filter/CreateLogfeederFilter_tmpl.html | 1 +
17 files changed, 87 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/OutputMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/OutputMgr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/OutputMgr.java
index c65b352..41b005b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/OutputMgr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/OutputMgr.java
@@ -29,6 +29,7 @@ import java.util.UUID;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
import org.apache.ambari.logfeeder.logconfig.filter.FilterLogData;
import org.apache.ambari.logfeeder.output.Output;
import org.apache.log4j.Level;
@@ -86,7 +87,11 @@ public class OutputMgr {
if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) {
jsonObj.put("ip", LogFeederUtil.ipAddress);
}
-
+
+ //Add level
+ if (jsonObj.get("level") == null) {
+ jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
+ }
if (input.isUseEventMD5() || input.isGenEventMD5()) {
String prefix = "";
Object logtimeObj = jsonObj.get("logtime");
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
index b069029..09673a0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
@@ -22,6 +22,7 @@ public class LogFeederConstants {
public static final String ALL = "all";
public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config";
+ public static final String LOG_LEVEL_UNKNOWN = "UNKNOWN";
// solr fields
public static final String SOLR_LEVEL = "level";
public static final String SOLR_COMPONENT = "type";
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
index 593be65..43a7eb7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
@@ -27,6 +27,9 @@ public class LogSearchConstants {
public static final String ERROR = "ERROR";
public static final String TRACE = "TRACE";
public static final String FATAL = "FATAL";
+ public static final String UNKNOWN = "UNKNOWN";
+
+ public static final String[] SUPPORTED_LOG_LEVEL ={FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN};
// Application Constants
public static final String HOST = "H";
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
index 92baaff..361f8e3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
@@ -188,15 +188,8 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
}
}
if (xAxisField.equalsIgnoreCase(LogSearchConstants.SOLR_LEVEL)) {
- List<String> logLevels = new ArrayList<String>();
- logLevels.add(LogSearchConstants.FATAL);
- logLevels.add(LogSearchConstants.ERROR);
- logLevels.add(LogSearchConstants.WARN);
- logLevels.add(LogSearchConstants.INFO);
- logLevels.add(LogSearchConstants.DEBUG);
- logLevels.add(LogSearchConstants.TRACE);
Collection<VNameValue> sortedVNameValues = new ArrayList<VNameValue>();
- for (String level : logLevels) {
+ for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
VNameValue value = new VNameValue();
value.setName(level);
String val = "0";
@@ -292,17 +285,10 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField)
&& LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(xAxisField)) {
Collection<VBarGraphData> levelVGraphData = dataList.getGraphData();
- List<String> logLevels = new ArrayList<String>();
- logLevels.add(LogSearchConstants.FATAL);
- logLevels.add(LogSearchConstants.ERROR);
- logLevels.add(LogSearchConstants.WARN);
- logLevels.add(LogSearchConstants.INFO);
- logLevels.add(LogSearchConstants.DEBUG);
- logLevels.add(LogSearchConstants.TRACE);
for (VBarGraphData garphData : levelVGraphData) {
Collection<VNameValue> valueList = garphData.getDataCount();
Collection<VNameValue> valueListSorted = new ArrayList<VNameValue>();
- for (String level : logLevels) {
+ for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
String val = "0";
for (VNameValue value : valueList) {
if (value.getName().equalsIgnoreCase(level)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
index a2c15f4..748d2f9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
@@ -540,84 +540,25 @@ public class LogsMgr extends MgrBase {
HashMap<String, String> map = new HashMap<String, String>();
List<VNameValue> logsCounts = new ArrayList<VNameValue>();
try {
-
queryGenerator.setFacetField(query, LogSearchConstants.SOLR_LEVEL);
-
List<Count> logLevelCounts = getFacetCounts(query,
- LogSearchConstants.SOLR_LEVEL);
- if(logLevelCounts == null){
+ LogSearchConstants.SOLR_LEVEL);
+ if (logLevelCounts == null) {
return logsCounts;
}
for (Count count : logLevelCounts) {
map.put(count.getName().toUpperCase(), "" + count.getCount());
}
- String level = LogSearchConstants.FATAL;
- VNameValue nameValue = null;
-
- String value = map.get(level);
- if (stringUtil.isEmpty(value)){
- value = defalutValue;
- }
- nameValue = new VNameValue();
- nameValue.setName(level);
- nameValue.setValue(value);
- logsCounts.add(nameValue);
-
- level = LogSearchConstants.ERROR;
-
- value = map.get(level);
- if (stringUtil.isEmpty(value)){
- value = defalutValue;
- }
- nameValue = new VNameValue();
- nameValue.setName(level);
- nameValue.setValue(value);
- logsCounts.add(nameValue);
-
- level = LogSearchConstants.WARN;
-
- value = map.get(level);
- if (stringUtil.isEmpty(value)){
- value = defalutValue;
- }
- nameValue = new VNameValue();
- nameValue.setName(level);
- nameValue.setValue(value);
- logsCounts.add(nameValue);
-
- level = LogSearchConstants.INFO;
-
- value = map.get(level);
- if (stringUtil.isEmpty(value)){
- value = defalutValue;
- }
- nameValue = new VNameValue();
- nameValue.setName(level);
- nameValue.setValue(value);
- logsCounts.add(nameValue);
-
- level = LogSearchConstants.DEBUG;
-
- value = map.get(level);
- if (stringUtil.isEmpty(value)){
- value = defalutValue;
- }
- nameValue = new VNameValue();
- nameValue.setName(level);
- nameValue.setValue(value);
- logsCounts.add(nameValue);
-
- level = LogSearchConstants.TRACE;
-
- value = map.get(level);
- if (stringUtil.isEmpty(value)){
- value = defalutValue;
+ for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
+ VNameValue nameValue = new VNameValue();
+ String value = map.get(level);
+ if (stringUtil.isEmpty(value)) {
+ value = defalutValue;
+ }
+ nameValue.setName(level);
+ nameValue.setValue(value);
+ logsCounts.add(nameValue);
}
- nameValue = new VNameValue();
- nameValue.setName(level);
- nameValue.setValue(value);
- logsCounts.add(nameValue);
-
} catch (SolrException | SolrServerException | IOException e) {
logger.error("Error during solrQuery=" + query, e);
}
@@ -1260,7 +1201,6 @@ public class LogsMgr extends MgrBase {
String unit = getUnit((String) searchCriteria.getParamValue("unit"));
List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
- List<String> logLevels = ConfigUtil.logLevels;
String jsonHistogramQuery = queryGenerator
.buildJSONFacetTermTimeRangeQuery(
@@ -1287,7 +1227,7 @@ public class LogsMgr extends MgrBase {
Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
List<VBarGraphData> graphDatas = new ArrayList<VBarGraphData>();
- for (String level : logLevels) {
+ for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
boolean isLevelPresent = false;
VBarGraphData vData1 = null;
for (VBarGraphData vData2 : histogramData) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
index 2bf3b51..bdd304f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
@@ -19,7 +19,6 @@
package org.apache.ambari.logsearch.util;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -35,8 +34,6 @@ import org.codehaus.jettison.json.JSONObject;
public class ConfigUtil {
static Logger logger = Logger.getLogger(MgrBase.class);
- public static List<String> logLevels = new ArrayList<String>();
-
public static HashMap<String, String> serviceLogsColumnMapping = new HashMap<String, String>();
public static HashMap<String, String> auditLogsColumnMapping = new HashMap<String, String>();
@@ -44,7 +41,6 @@ public class ConfigUtil {
public static HashMap<String, String> schemaFieldsName = new HashMap<String, String>();
public static void initializeApplicationConfig() {
- intializeLogLevels();
initializeColumnMapping();
}
@@ -67,16 +63,6 @@ public class ConfigUtil {
}
}
}
-
- private static void intializeLogLevels() {
- logLevels.add(LogSearchConstants.TRACE);
- logLevels.add(LogSearchConstants.DEBUG);
- logLevels.add(LogSearchConstants.INFO);
- logLevels.add(LogSearchConstants.WARN);
- logLevels.add(LogSearchConstants.ERROR);
- logLevels.add(LogSearchConstants.FATAL);
- }
-
private static void initializeColumnMapping() {
String serviceLogsColumnMappingArray[] = PropertiesUtil
.getPropertyStringList("logsearch.solr.service.logs.column.mapping");
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
index cb8ca4e..e88afea 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
@@ -172,7 +172,7 @@ define(['require',
content: view,
viewType: 'Filter',
resizable: false,
- width: 950,
+ width: 1000,
height: 550,
autoFocus1stElement : false,
buttons: [{
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
index d2ee3d8..0e7f1b8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
@@ -44,6 +44,7 @@ define(['require',
DEBUG : "[data-id='DEBUG']",
FATAL : "[data-id='FATAL']",
TRACE : "[data-id='TRACE']",
+ UNKNOWN : "[data-id='UNKNOWN']",
loader:".server-info .fa-spin"
},
@@ -80,7 +81,7 @@ define(['require',
}
},
fetchLogLevelCounts : function(params){
- $.extend(this.logLevelList.queryParams,params,{level: "FATAL,ERROR,WARN,INFO,DEBUG,TRACE"});
+ $.extend(this.logLevelList.queryParams,params,{level: "FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN"});
this.ui.loader.show();
this.logLevelList.fetch({reset:true});
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelView.js
index 17a67fb..79c8bcf 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelView.js
@@ -45,6 +45,7 @@ define(['require',
DEBUG : "[data-id='DEBUG']",
FATAL : "[data-id='FATAL']",
TRACE : "[data-id='TRACE']",
+ UNKNOWN : "[data-id='UNKNOWN']",
togglePieViewButton:'#logToggle',
pieRegionId:'#r_logLevelPieChart',
logTable:'#logTable'
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
index 95324d9..9bdf0fa 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
@@ -81,7 +81,7 @@ define(['require',
this.model = new VUserFilter();
this.levelCollection = new Backbone.Collection();
- var levelArr = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"];
+ var levelArr = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE", "UNKNOWN"];
for (var i in levelArr) {
this.levelCollection.add(new Backbone.Model({ type: levelArr[i] }));
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
index 76d0537..35d0c9a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
@@ -255,7 +255,8 @@ define(['require',
(model.get('name') === 'WARN') ? ("#FF8916") :
(model.get('name') === 'FATAL') ? ("#830A0A") :
(model.get('name') === 'DEBUG') ? ("#65E8FF") :
- (model.get('name') === 'TRACE') ? ("#888888") : ("white"));
+ (model.get('name') === 'TRACE') ? ("#888888") :
+ (model.get('name') === 'UNKNOWN') ? ("#bdbdbd") : ("white"));
} else {
Obj['color'] = color[i];
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
index 47393bb..ae76ba8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
@@ -203,7 +203,8 @@ define(['require',
( (""+model.get('name')).toUpperCase() === 'WARN') ? ("#FF8916") :
( (""+model.get('name')).toUpperCase() === 'FATAL') ? ("#830A0A") :
( (""+model.get('name')).toUpperCase() === 'DEBUG') ? ("#65E8FF") :
- ( (""+model.get('name')).toUpperCase() === 'TRACE') ? ("#888888") : color[i]);
+ ( (""+model.get('name')).toUpperCase() === 'TRACE') ? ("#888888") :
+ ( (""+model.get('name')).toUpperCase() === 'UNKNOWN') ? ("#bdbdbd") : color[i]);
} else {
Obj['color'] = color[i];
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
index 4b42eab..0b32237 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
@@ -495,7 +495,7 @@ define(['require',
direction: "descending",
orderable: true,
displayOrder: 1,
- width: "10",
+ width: "17",
className: "logTime",
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
index 66f851c..a6445c6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
@@ -369,7 +369,8 @@ define(['backbone',
( (""+name).toUpperCase() === 'WARN') ? ("#FF8916") :
( (""+name).toUpperCase() === 'FATAL') ? ("#830A0A") :
( (""+name).toUpperCase() === 'DEBUG') ? ("#65E8FF") :
- ( (""+name).toUpperCase() === 'TRACE') ? ("#888888") : "")
+ ( (""+name).toUpperCase() === 'TRACE') ? ("#888888") :
+ ( (""+name).toUpperCase() === 'UNKNOWN') ? ("#bdbdbd") : "")
});
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
index 240cc4d..543533b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css
@@ -261,7 +261,9 @@ p.log-line:before {
.bubbleGraph .label,.node--root,.node--leaf {
pointer-events: none;
}
-
+.UNKNOWN {
+ fill: #bdbdbd;
+}
.TRACE {
fill: #888888;
}
@@ -489,6 +491,9 @@ div.columnmanager-visibilitycontrol {
height: 100%;
float: left;
}
+.node.UNKNOWN {
+ background-color: #bdbdbd;
+}
.node.TRACE {
background-color: #888888;
@@ -514,6 +519,9 @@ div.columnmanager-visibilitycontrol {
background-color: #830A0A;
}
+.label-UNKNOWN {
+ background-color: #bdbdbd;
+}
.label-TRACE {
background-color: #888888;
}
@@ -589,6 +597,15 @@ div.columnmanager-visibilitycontrol {
top: -3px;
right: 0;
}
+.server-info a.node.UNKNOWN {
+ border-color: #bdbdbd;
+ color: #bdbdbd;
+}
+
+.server-info a.node.UNKNOWN.active {
+ background-color: #bdbdbd;
+ color: #fff;
+}
.server-info a.node.TRACE {
border-color: #888888;
@@ -649,6 +666,9 @@ div.columnmanager-visibilitycontrol {
background-color: #830A0A;
color: #fff;
}
+.UNKNOWN {
+ color: #bdbdbd;
+}
.TRACE {
color: #888888;
@@ -2348,19 +2368,19 @@ input:-webkit-autofill,input:-webkit-autofill:hover,input:-webkit-autofill:focus
}
.logFeederTable {
- width: 906px;
+ width: 965px;
display: block;
}
.logFeederTable tbody {
height: 365px;
overflow-y: auto;
- width: 906px;
+ width: 965px;
display: block;
}
.logFeederTable th {
- width: 90px;
+ width: 106px;
border-left: 1px solid #CCC;
border-right: 1px solid #CCC;
border-top: 1px solid #CCC !important;
@@ -2376,7 +2396,7 @@ input:-webkit-autofill,input:-webkit-autofill:hover,input:-webkit-autofill:focus
}
.logFeederTable th:nth-child(2) {
- width: 190px;
+ width: 165px;
border-top: 1px solid #CCC !important;
border-bottom: 1px solid #CCC !important;
}
@@ -2391,12 +2411,12 @@ input:-webkit-autofill,input:-webkit-autofill:hover,input:-webkit-autofill:focus
}
.logFeederTable td:first-of-type {
- width: 269px;
+ width: 240px;
padding: 0 10px;
}
.logFeederTable td:nth-child(2) {
- width: 178px;
+ width: 135px;
padding: 0 10px;
}
@@ -2929,4 +2949,16 @@ button.defaultCancelBtn{
}
.comparisonTab .daterangepicker .ranges{
float: right !important;
+}
+
+/*------------New Log Levels Tab CSS--------------*/
+.levelBox{
+ width: 13%;
+ margin: 0 .6%;
+ float: left;
+ position: relative;
+ min-height: 1px;
+}
+.levelBox:last-of-type{
+ clear : right;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/dashboard/LogLevelBoxView_tmpl.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/dashboard/LogLevelBoxView_tmpl.html b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/dashboard/LogLevelBoxView_tmpl.html
index 4c3cd8d..0424cc0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/dashboard/LogLevelBoxView_tmpl.html
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/dashboard/LogLevelBoxView_tmpl.html
@@ -16,29 +16,34 @@
-->
<div class="server-info">
<a href="javascript:void(0);"
- class="col-md-2 node FATAL text-center active"> <strong
+ class="levelBox node FATAL text-center active"> <strong
data-total="FATAL">0</strong><span>Fatal</span> <i
class="fa fa-spinner fa-spin"></i>
</a> <a href="javascript:void(0);"
- class="col-md-2 node ERROR text-center active"> <strong
+ class="levelBox node ERROR text-center active"> <strong
data-total="ERROR">0</strong><span>Error</span> <i
class="fa fa-spinner fa-spin"></i>
</a> <a href="javascript:void(0);"
- class="col-md-2 node WARN text-center active"> <strong
+ class="levelBox node WARN text-center active"> <strong
data-total="WARN">0</strong><span>Warn</span> <i
class="fa fa-spinner fa-spin"></i>
</a> <a href="javascript:void(0);"
- class="col-md-2 node INFO text-center active"> <strong
+ class="levelBox node INFO text-center active"> <strong
data-total="INFO">0</strong><span>Info</span> <i
class="fa fa-spinner fa-spin"></i>
</a> <a href="javascript:void(0);"
- class="col-md-2 node DEBUG text-center active"> <strong
+ class="levelBox node DEBUG text-center active"> <strong
data-total="DEBUG">0</strong><span>Debug</span> <i
class="fa fa-spinner fa-spin"></i>
</a> <a href="javascript:void(0);"
- class="col-md-2 node TRACE text-center active"> <strong
+ class="levelBox node TRACE text-center active"> <strong
data-total="TRACE">0</strong><span>Trace</span> <i
class="fa fa-spinner fa-spin"></i>
</a>
+ <a href="javascript:void(0);"
+ class="levelBox node UNKNOWN text-center active"> <strong
+ data-total="UNKNOWN">0</strong><span>Unknown</span> <i
+ class="fa fa-spinner fa-spin"></i>
+ </a>
</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d8bb21c2/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/filter/CreateLogfeederFilter_tmpl.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/filter/CreateLogfeederFilter_tmpl.html b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/filter/CreateLogfeederFilter_tmpl.html
index fe2a5c1..a4d78fa 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/filter/CreateLogfeederFilter_tmpl.html
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/filter/CreateLogfeederFilter_tmpl.html
@@ -34,6 +34,7 @@
<th class="INFO text-center"><span class="pull-left"><input type="checkbox" data-value="INFO"></span><span>INFO</span></th>
<th class="DEBUG text-center"><span class="pull-left"><input type="checkbox" data-value="DEBUG"></span><span>DEBUG</span></th>
<th class="TRACE text-center"><span class="pull-left"><input type="checkbox" data-value="TRACE"></span><span>TRACE</span></th>
+ <th class="UNKNOWN text-center"><span class="pull-left"><input type="checkbox" data-value="UNKNOWN"></span><span>UNKNOWN</span></th>
</tr>
</thead>
<tbody id="filterContent"></tbody>
[23/55] [abbrv] ambari git commit: AMBARI-18209. UI: add missing unit
tests for models (alexantonenko)
Posted by ol...@apache.org.
AMBARI-18209. UI: add missing unit tests for models (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ad19662
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ad19662
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ad19662
Branch: refs/heads/branch-dev-logsearch
Commit: 9ad19662817e2c24a4c3e72bb465b0a35e521bf2
Parents: 2ad7c80
Author: Alex Antonenko <hi...@gmail.com>
Authored: Fri Aug 19 14:42:46 2016 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Fri Aug 19 15:29:17 2016 +0300
----------------------------------------------------------------------
ambari-web/app/assets/test/tests.js | 8 +
ambari-web/app/models/client_component.js | 8 +-
ambari-web/app/models/cluster.js | 2 +-
ambari-web/app/models/master_component.js | 2 +-
ambari-web/app/models/root_service.js | 2 +-
ambari-web/app/models/slave_component.js | 2 +-
ambari-web/app/models/widget_property.js | 11 +-
ambari-web/test/models/client_component_test.js | 115 +++++++
ambari-web/test/models/cluster_test.js | 89 ++++++
ambari-web/test/models/master_component_test.js | 59 ++++
ambari-web/test/models/root_service_test.js | 78 +++++
ambari-web/test/models/slave_component_test.js | 59 ++++
ambari-web/test/models/stack_test.js | 226 +++++++++++++
ambari-web/test/models/widget_property_test.js | 320 +++++++++++++++++++
ambari-web/test/models/widget_test.js | 70 ++++
15 files changed, 1036 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/assets/test/tests.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/assets/test/tests.js b/ambari-web/app/assets/test/tests.js
index 364f2ed..7dbac75 100644
--- a/ambari-web/app/assets/test/tests.js
+++ b/ambari-web/app/assets/test/tests.js
@@ -373,17 +373,25 @@ var files = [
'test/models/alerts/alert_group_test',
'test/models/alerts/alert_instance_test',
'test/models/authentication_test',
+ 'test/models/client_component_test',
'test/models/cluster_states_test',
+ 'test/models/cluster_test',
'test/models/form_test',
'test/models/host_test',
'test/models/host_component_test',
'test/models/hosts_test',
+ 'test/models/master_component_test',
'test/models/operating_system_test',
'test/models/repository_test',
+ 'test/models/root_service_test',
'test/models/stack_service_component_test',
'test/models/service_test',
+ 'test/models/slave_component_test',
+ 'test/models/stack_test',
'test/models/stack_service_test',
'test/models/user_test',
+ 'test/models/widget_test',
+ 'test/models/widget_property_test',
'test/models/host_stack_version_test',
'test/models/upgrade_entity_test',
'test/models/finished_upgrade_entity_test',
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/models/client_component.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/client_component.js b/ambari-web/app/models/client_component.js
index 19ef7ce..50a3bb6 100644
--- a/ambari-web/app/models/client_component.js
+++ b/ambari-web/app/models/client_component.js
@@ -37,19 +37,19 @@ App.ClientComponent = DS.Model.extend({
*
* @type {boolean}
*/
- allowToDelete: function() {
+ allowToDelete: function () {
return this.get('totalCount') === (this.get('installedCount') + this.get('installFailedCount') + this.get('initCount') + this.get('unknownCount'));
}.property('totalCount', 'installedCount', 'installFailedCount', 'initCount', 'unknownCount'),
- summaryLabelClassName:function(){
+ summaryLabelClassName: function () {
return 'label_for_'+this.get('componentName').toLowerCase();
}.property('componentName'),
- summaryValueClassName:function(){
+ summaryValueClassName: function () {
return 'value_for_'+this.get('componentName').toLowerCase();
}.property('componentName'),
- displayNamePluralized: function() {
+ displayNamePluralized: function () {
return stringUtils.pluralize(this.get('installedCount'), this.get('displayName'));
}.property('installedCount')
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/models/cluster.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/cluster.js b/ambari-web/app/models/cluster.js
index a571a28..57f91ee 100644
--- a/ambari-web/app/models/cluster.js
+++ b/ambari-web/app/models/cluster.js
@@ -35,7 +35,7 @@ App.Cluster = DS.Model.extend({
isKerberosEnabled: Em.computed.equal('securityType', 'KERBEROS'),
- isCredentialStorePersistent: function() {
+ isCredentialStorePersistent: function () {
return this.get('credentialStoreProperties')[credentialUtils.STORE_TYPES.PERSISTENT_PATH] === "true";
}.property('credentialStoreProperties')
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/models/master_component.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/master_component.js b/ambari-web/app/models/master_component.js
index ee44d0d..a4f3717 100644
--- a/ambari-web/app/models/master_component.js
+++ b/ambari-web/app/models/master_component.js
@@ -20,7 +20,7 @@ var App = require('app');
var stringUtils = require('utils/string_utils');
App.MasterComponent = App.ClientComponent.extend({
- displayNamePluralized: function() {
+ displayNamePluralized: function () {
return stringUtils.pluralize(this.get('startedCount'), this.get('displayName'));
}.property('startedCount')
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/models/root_service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/root_service.js b/ambari-web/app/models/root_service.js
index ff73675..06ca8d0 100644
--- a/ambari-web/app/models/root_service.js
+++ b/ambari-web/app/models/root_service.js
@@ -31,7 +31,7 @@ App.RootService = DS.Model.extend({
App.RootService.FIXTURES = [];
-App.RootServiceComponents = DS.Model.extend({
+App.RootServiceComponents = DS.Model.extend({
componentName: DS.attr('string'),
displayName: Em.computed.formatRole('componentName', false),
componentVersion: DS.attr('string'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/models/slave_component.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/slave_component.js b/ambari-web/app/models/slave_component.js
index eb20c75..a108678 100644
--- a/ambari-web/app/models/slave_component.js
+++ b/ambari-web/app/models/slave_component.js
@@ -20,7 +20,7 @@ var App = require('app');
var stringUtils = require('utils/string_utils');
App.SlaveComponent = App.ClientComponent.extend({
- displayNamePluralized: function() {
+ displayNamePluralized: function () {
return stringUtils.pluralize(this.get('startedCount'), this.get('displayName'));
}.property('startedCount')
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/app/models/widget_property.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/widget_property.js b/ambari-web/app/models/widget_property.js
index 0b7e414..6b5dc82 100644
--- a/ambari-web/app/models/widget_property.js
+++ b/ambari-web/app/models/widget_property.js
@@ -73,13 +73,10 @@ App.WidgetProperty = Ember.Object.extend({
/**
* Define whether property is valid
- * Computed property
- * Should be defined in child class
+ * Computed property should be defined in child class
* @type {Boolean}
*/
- isValid: function () {
- return true;
- }.property(),
+ isValid: true,
/**
* Define whether property is required by user
@@ -96,7 +93,7 @@ App.WidgetPropertyTypes = [
displayType: 'textField',
classNames: 'widget-property-unit',
isValid: function () {
- return this.get('isRequired') ? this.get('value') : true;
+ return this.get('isRequired') ? Boolean(this.get('value')) : true;
}.property('value'),
valueMap: {
"value": "display_unit"
@@ -214,7 +211,7 @@ App.WidgetPropertyTypes = [
*/
validate: function (value) {
value = Number(('' + value).trim());
- if (!value) {
+ if (!value && !isNaN(value)) {
return true;
}
return validator.isValidFloat(value) && value > this.get('MIN_VALUE') && value <= this.get('MAX_VALUE');
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/client_component_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/client_component_test.js b/ambari-web/test/models/client_component_test.js
new file mode 100644
index 0000000..5f92b96
--- /dev/null
+++ b/ambari-web/test/models/client_component_test.js
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/client_component');
+
+describe('App.ClientComponent', function () {
+
+ var clientComponent;
+
+ beforeEach(function () {
+ clientComponent = App.ClientComponent.createRecord();
+ });
+
+ describe('#allowToDelete', function () {
+
+ var cases = [
+ {
+ statesCounts: {
+ installedCount: 1,
+ installFailedCount: 2,
+ initCount: 3,
+ unknownCount: 4,
+ totalCount: 10
+ },
+ allowToDelete: true,
+ title: 'delete allowed'
+ },
+ {
+ statesCounts: {
+ installedCount: 1,
+ installFailedCount: 2,
+ initCount: 3,
+ unknownCount: 4,
+ totalCount: 11
+ },
+ allowToDelete: false,
+ title: 'delete not allowed'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ clientComponent.setProperties(item.statesCounts);
+ expect(clientComponent.get('allowToDelete')).to.equal(item.allowToDelete);
+ });
+
+ });
+
+ });
+
+ describe('#summaryLabelClassName', function () {
+
+ it('should use lower case of component name', function () {
+ clientComponent.set('componentName', 'ZOOKEEPER_CLIENT');
+ expect(clientComponent.get('summaryLabelClassName')).to.equal('label_for_zookeeper_client');
+ });
+
+ });
+
+ describe('#summaryValueClassName', function () {
+
+ it('should use lower case of component name', function () {
+ clientComponent.set('componentName', 'ZOOKEEPER_CLIENT');
+ expect(clientComponent.get('summaryValueClassName')).to.equal('value_for_zookeeper_client');
+ });
+
+ });
+
+ describe('#displayNamePluralized', function () {
+
+ var cases = [
+ {
+ installedCount: 1,
+ displayNamePluralized: 'Zookeeper Client',
+ title: 'singular'
+ },
+ {
+ installedCount: 2,
+ displayNamePluralized: 'Zookeeper Clients',
+ title: 'plural'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ clientComponent.setProperties({
+ displayName: 'Zookeeper Client',
+ installedCount: item.installedCount
+ });
+ expect(clientComponent.get('displayNamePluralized')).to.equal(item.displayNamePluralized);
+ });
+
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/cluster_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/cluster_test.js b/ambari-web/test/models/cluster_test.js
new file mode 100644
index 0000000..a6bafba
--- /dev/null
+++ b/ambari-web/test/models/cluster_test.js
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/cluster');
+
+describe('App.Cluster', function () {
+
+ var cluster;
+
+ beforeEach(function () {
+ cluster = App.Cluster.createRecord();
+ });
+
+ describe('#isKerberosEnabled', function () {
+
+ var cases = [
+ {
+ securityType: 'KERBEROS',
+ isKerberosEnabled: true,
+ title: 'Kerberos enabled'
+ },
+ {
+ securityType: 'NONE',
+ isKerberosEnabled: false,
+ title: 'Kerberos disabled'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ cluster.set('securityType', item.securityType);
+ expect(cluster.get('isKerberosEnabled')).to.equal(item.isKerberosEnabled);
+ });
+
+ });
+
+ describe('#isCredentialStorePersistent', function () {
+
+ var cases = [
+ {
+ propertyValue: 'false',
+ isCredentialStorePersistent: false,
+ title: 'no persistent credential store'
+ },
+ {
+ propertyValue: true,
+ isCredentialStorePersistent: false,
+ title: 'malformed value'
+ },
+ {
+ propertyValue: 'true',
+ isCredentialStorePersistent: true,
+ title: 'persistent credential store'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ cluster.set('credentialStoreProperties', {
+ 'storage.persistent': item.propertyValue
+ });
+ expect(cluster.get('isCredentialStorePersistent')).to.equal(item.isCredentialStorePersistent);
+ });
+
+ });
+
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/master_component_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/master_component_test.js b/ambari-web/test/models/master_component_test.js
new file mode 100644
index 0000000..2bd4e09
--- /dev/null
+++ b/ambari-web/test/models/master_component_test.js
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/master_component');
+
+describe('App.MasterComponent', function () {
+
+ var masterComponent;
+
+ beforeEach(function () {
+ masterComponent = App.MasterComponent.createRecord();
+ });
+
+ describe('#displayNamePluralized', function () {
+
+ var cases = [
+ {
+ startedCount: 1,
+ displayNamePluralized: 'Zookeeper Server',
+ title: 'singular'
+ },
+ {
+ startedCount: 2,
+ displayNamePluralized: 'Zookeeper Servers',
+ title: 'plural'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ masterComponent.setProperties({
+ displayName: 'Zookeeper Server',
+ startedCount: item.startedCount
+ });
+ expect(masterComponent.get('displayNamePluralized')).to.equal(item.displayNamePluralized);
+ });
+
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/root_service_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/root_service_test.js b/ambari-web/test/models/root_service_test.js
new file mode 100644
index 0000000..fa4a646
--- /dev/null
+++ b/ambari-web/test/models/root_service_test.js
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/root_service');
+
+var roleMock = function (str, flag) {
+ return flag ? str.toLowerCase() : str.toCapital();
+};
+
+describe('App.RootService', function () {
+
+ var rootService;
+
+ beforeEach(function () {
+ rootService = App.RootService.createRecord();
+ });
+
+ describe('#displayName', function () {
+
+ beforeEach(function () {
+ sinon.stub(App.format, 'role', roleMock);
+ });
+
+ afterEach(function () {
+ App.format.role.restore();
+ });
+
+ it('should format service name', function () {
+ rootService.set('serviceName', 'HDFS');
+ expect(rootService.get('displayName')).to.equal('hdfs');
+ });
+
+ });
+
+});
+
+describe('App.RootServiceComponents', function () {
+
+ var rootServiceComponents;
+
+ beforeEach(function () {
+ rootServiceComponents = App.RootServiceComponents.createRecord();
+ });
+
+ describe('#displayName', function () {
+
+ beforeEach(function () {
+ sinon.stub(App.format, 'role', roleMock);
+ });
+
+ afterEach(function () {
+ App.format.role.restore();
+ });
+
+ it('should format component name', function () {
+ rootServiceComponents.set('componentName', 'DATANODE');
+ expect(rootServiceComponents.get('displayName')).to.equal('Datanode');
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/slave_component_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/slave_component_test.js b/ambari-web/test/models/slave_component_test.js
new file mode 100644
index 0000000..cf21aef
--- /dev/null
+++ b/ambari-web/test/models/slave_component_test.js
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/slave_component');
+
+describe('App.SlaveComponent', function () {
+
+ var slaveComponent;
+
+ beforeEach(function () {
+ slaveComponent = App.SlaveComponent.createRecord();
+ });
+
+ describe('#displayNamePluralized', function () {
+
+ var cases = [
+ {
+ startedCount: 1,
+ displayNamePluralized: 'DataNode',
+ title: 'singular'
+ },
+ {
+ startedCount: 2,
+ displayNamePluralized: 'DataNodes',
+ title: 'plural'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ slaveComponent.setProperties({
+ displayName: 'DataNode',
+ startedCount: item.startedCount
+ });
+ expect(slaveComponent.get('displayNamePluralized')).to.equal(item.displayNamePluralized);
+ });
+
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/stack_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/stack_test.js b/ambari-web/test/models/stack_test.js
new file mode 100644
index 0000000..7f1f467
--- /dev/null
+++ b/ambari-web/test/models/stack_test.js
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/stack');
+
+describe('App.Stack', function () {
+
+ var stack;
+
+ beforeEach(function () {
+ stack = App.Stack.createRecord();
+ });
+
+ describe('#stackNameVersion', function () {
+
+ it('should concat stack name and stack version', function () {
+ stack.setProperties({
+ stackName: 'HDP',
+ stackVersion: '2.5'
+ });
+ expect(stack.get('stackNameVersion')).to.equal('HDP-2.5');
+ });
+
+ });
+
+ describe('#isPatch', function () {
+
+ var cases = [
+ {
+ type: 'PATCH',
+ isPatch: true
+ },
+ {
+ type: 'STANDARD',
+ isPatch: false
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.type, function () {
+ stack.set('type', item.type);
+ expect(stack.get('isPatch')).to.equal(item.isPatch);
+ });
+
+ });
+
+ });
+
+ describe('#displayName', function () {
+
+ it('should concat stack name and stack version', function () {
+ stack.setProperties({
+ stackName: 'HDP',
+ repositoryVersion: '2.5.0.0-999'
+ });
+ expect(stack.get('displayName')).to.equal('HDP-2.5.0.0-999');
+ });
+
+ });
+
+ describe('#repositories', function () {
+
+ beforeEach(function () {
+ stack.reopen({
+ operatingSystems: [
+ Em.Object.create({
+ isSelected: false,
+ repositories: [
+ {
+ id: 0
+ },
+ {
+ id: 1
+ }
+ ]
+ }),
+ Em.Object.create({
+ isSelected: false,
+ repositories: [
+ {
+ id: 2
+ },
+ {
+ id: 3
+ }
+ ]
+ }),
+ Em.Object.create({
+ isSelected: false,
+ repositories: [
+ {
+ id: 4
+ },
+ {
+ id: 5
+ }
+ ]
+ })
+ ]
+ });
+ });
+
+ it('no OSes selected', function () {
+ expect(stack.get('repositories')).to.be.empty;
+ });
+
+ it('some OSes selected', function () {
+ stack.get('operatingSystems')[0].isSelected = true;
+ stack.get('operatingSystems')[2].isSelected = true;
+ expect(stack.get('repositories').toArray()).to.eql([
+ {
+ id: 0
+ },
+ {
+ id: 1
+ },
+ {
+ id: 4
+ },
+ {
+ id: 5
+ }
+ ]);
+ });
+
+ });
+
+ describe('#cleanReposBaseUrls', function () {
+
+ beforeEach(function () {
+ stack.reopen({
+ operatingSystems: [
+ Em.Object.create({
+ isSelected: true,
+ repositories: [
+ Em.Object.create({
+ baseUrl: 'http://localhost/repo0'
+ }),
+ Em.Object.create({
+ baseUrl: 'http://localhost/repo1'
+ })
+ ]
+ }),
+ Em.Object.create({
+ isSelected: true,
+ repositories: [
+ Em.Object.create({
+ baseUrl: 'http://localhost/repo2'
+ }),
+ Em.Object.create({
+ baseUrl: 'http://localhost/repo3'
+ })
+ ]
+ })
+ ]
+ });
+ stack.cleanReposBaseUrls();
+ });
+
+ it('should clear repo urls', function () {
+ expect(stack.get('repositories').mapProperty('baseUrl')).to.eql(['', '', '', '']);
+ });
+
+ });
+
+ describe('#restoreReposBaseUrls', function () {
+
+ beforeEach(function () {
+ stack.reopen({
+ operatingSystems: [
+ Em.Object.create({
+ isSelected: true,
+ repositories: [
+ Em.Object.create({
+ baseUrlInit: 'http://localhost/repo0'
+ }),
+ Em.Object.create({
+ baseUrlInit: 'http://localhost/repo1'
+ })
+ ]
+ }),
+ Em.Object.create({
+ isSelected: true,
+ repositories: [
+ Em.Object.create({
+ baseUrlInit: 'http://localhost/repo2'
+ }),
+ Em.Object.create({
+ baseUrlInit: 'http://localhost/repo3'
+ })
+ ]
+ })
+ ]
+ });
+ stack.restoreReposBaseUrls();
+ });
+
+ it('should reset repo urls', function () {
+ expect(stack.get('repositories').mapProperty('baseUrl')).to.eql([
+ 'http://localhost/repo0',
+ 'http://localhost/repo1',
+ 'http://localhost/repo2',
+ 'http://localhost/repo3'
+ ]);
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/widget_property_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/widget_property_test.js b/ambari-web/test/models/widget_property_test.js
new file mode 100644
index 0000000..ee1cdb4
--- /dev/null
+++ b/ambari-web/test/models/widget_property_test.js
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/widget_property');
+
+describe('App.WidgetProperty', function () {
+
+ var widgetProperty,
+ unit = App.WidgetPropertyTypes.findProperty('name', 'display_unit'),
+ threshold = App.WidgetPropertyTypes.findProperty('name', 'threshold'),
+ validate = function (value) {
+ return !isNaN(value);
+ };
+
+ beforeEach(function () {
+ widgetProperty = App.WidgetProperty.create();
+ });
+
+ describe('#viewClass', function () {
+
+ var cases = [
+ {
+ displayType: 'textField',
+ viewClass: App.WidgetPropertyTextFieldView
+ },
+ {
+ displayType: 'threshold',
+ viewClass: App.WidgetPropertyThresholdView
+ },
+ {
+ displayType: 'select',
+ viewClass: App.WidgetPropertySelectView
+ },
+ {
+ displayType: 'none',
+ viewClass: undefined
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.displayType, function () {
+ widgetProperty.set('displayType', item.displayType);
+ expect(widgetProperty.get('viewClass')).to.eql(item.viewClass);
+ });
+
+ });
+
+ });
+
+ describe('#isValid', function () {
+
+ describe('display_unit', function () {
+
+ var cases = [
+ {
+ isRequired: true,
+ value: 'MB',
+ isValid: true,
+ title: 'valid value'
+ },
+ {
+ isRequired: true,
+ value: '0',
+ isValid: true,
+ title: 'non-empty value'
+ },
+ {
+ isRequired: true,
+ value: '',
+ isValid: false,
+ title: 'empty value'
+ },
+ {
+ isRequired: false,
+ value: '',
+ isValid: true,
+ title: 'value not required'
+ }
+ ];
+
+ beforeEach(function () {
+ widgetProperty.reopen(unit);
+ });
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ widgetProperty.setProperties({
+ isRequired: item.isRequired,
+ value: item.value
+ });
+ expect(widgetProperty.get('isValid')).to.equal(item.isValid);
+ });
+
+ });
+
+ });
+
+ describe('threshold', function () {
+
+ var cases = [
+ {
+ isSmallValueValid: true,
+ isBigValueValid: true,
+ isValid: true,
+ title: 'both threshold values are valid'
+ },
+ {
+ isSmallValueValid: false,
+ isBigValueValid: true,
+ isValid: false,
+ title: 'warning threshold value is invalid'
+ },
+ {
+ isSmallValueValid: true,
+ isBigValueValid: false,
+ isValid: false,
+ title: 'error threshold value is invalid'
+ },
+ {
+ isSmallValueValid: false,
+ isBigValueValid: false,
+ isValid: false,
+ title: 'both threshold values are invalid'
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ widgetProperty.reopen(threshold, {
+ isSmallValueValid: item.isSmallValueValid,
+ isBigValueValid: item.isBigValueValid
+ });
+ expect(widgetProperty.get('isValid')).to.equal(item.isValid);
+ });
+
+ });
+
+ });
+
+ });
+
+ describe('#isSmallValueValid', function () {
+
+ var cases = [
+ {
+ smallValue: '1',
+ isSmallValueValid: true,
+ title: 'valid value'
+ },
+ {
+ smallValue: 'value',
+ isSmallValueValid: false,
+ title: 'invalid value'
+ }
+ ];
+
+ beforeEach(function () {
+ widgetProperty.reopen(threshold);
+ sinon.stub(widgetProperty, 'validate', validate);
+ });
+
+ afterEach(function () {
+ widgetProperty.validate.restore();
+ });
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ widgetProperty.set('smallValue', item.smallValue);
+ expect(widgetProperty.get('isSmallValueValid')).to.equal(item.isSmallValueValid);
+ });
+
+ });
+
+ });
+
+ describe('#isBigValueValid', function () {
+
+ var cases = [
+ {
+ bigValue: '1',
+ isBigValueValid: true,
+ title: 'valid value'
+ },
+ {
+ bigValue: 'value',
+ isBigValueValid: false,
+ title: 'invalid value'
+ }
+ ];
+
+ beforeEach(function () {
+ widgetProperty.reopen(threshold);
+ sinon.stub(widgetProperty, 'validate', validate);
+ });
+
+ afterEach(function () {
+ widgetProperty.validate.restore();
+ });
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ widgetProperty.set('bigValue', item.bigValue);
+ expect(widgetProperty.get('isBigValueValid')).to.equal(item.isBigValueValid);
+ });
+
+ });
+
+ });
+
+ describe('#validate', function () {
+
+ var cases = [
+ {
+ value: '',
+ validateResult: true,
+ title: 'empty value'
+ },
+ {
+ value: ' \r\n\t ',
+ validateResult: true,
+ title: 'spaces only'
+ },
+ {
+ value: 'v',
+ validateResult: false,
+ title: 'invalid value'
+ },
+ {
+ value: ' v \r\n\t',
+ validateResult: false,
+ title: 'invalid value with spaces'
+ },
+ {
+ value: '-1',
+ validateResult: false,
+ title: 'value below the minimum'
+ },
+ {
+ value: ' -1 \r\n\t',
+ validateResult: false,
+ title: 'value below the minimum with spaces'
+ },
+ {
+ value: '2',
+ validateResult: false,
+ title: 'value above the minimum'
+ },
+ {
+ value: ' 2 \r\n\t',
+ validateResult: false,
+ title: 'value above the minimum with spaces'
+ },
+ {
+ value: '0,5',
+ validateResult: false,
+ title: 'malformed number'
+ },
+ {
+ value: ' 0,5 \r\n\t',
+ validateResult: false,
+ title: 'malformed number with spaces'
+ },
+ {
+ value: '0.5',
+ validateResult: true,
+ title: 'valid value'
+ },
+ {
+ value: ' 0.5 \r\n\t',
+ validateResult: true,
+ title: 'valid value with spaces'
+ },
+ {
+ value: '2E-1',
+ validateResult: true,
+ title: 'exponentially formatted value'
+ },
+ {
+ value: ' 2E-1 \r\n\t',
+ validateResult: true,
+ title: 'exponentially formatted value with spaces'
+ }
+ ];
+
+ beforeEach(function () {
+ widgetProperty.reopen(threshold);
+ });
+
+ cases.forEach(function (item) {
+
+ it(item.title, function () {
+ expect(widgetProperty.validate(item.value)).to.equal(item.validateResult);
+ });
+
+ });
+
+ });
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ad19662/ambari-web/test/models/widget_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/models/widget_test.js b/ambari-web/test/models/widget_test.js
new file mode 100644
index 0000000..589ecb9
--- /dev/null
+++ b/ambari-web/test/models/widget_test.js
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+require('models/widget');
+
+describe('App.Widget', function () {
+
+ var widget;
+
+ beforeEach(function () {
+ widget = App.Widget.createRecord();
+ });
+
+ describe('#viewClass', function () {
+
+ var cases = [
+ {
+ widgetType: 'GRAPH',
+ viewClass: App.GraphWidgetView
+ },
+ {
+ widgetType: 'TEMPLATE',
+ viewClass: App.TemplateWidgetView
+ },
+ {
+ widgetType: 'NUMBER',
+ viewClass: App.NumberWidgetView
+ },
+ {
+ widgetType: 'GAUGE',
+ viewClass: App.GaugeWidgetView
+ },
+ {
+ widgetType: 'HEATMAP',
+ viewClass: App.HeatmapWidgetView
+ },
+ {
+ widgetType: 'NONE',
+ viewClass: Em.View
+ }
+ ];
+
+ cases.forEach(function (item) {
+
+ it(item.widgetType, function () {
+ widget.set('widgetType', item.widgetType);
+ expect(widget.get('viewClass')).to.eql(item.viewClass);
+ });
+
+ });
+
+ });
+
+});
\ No newline at end of file
[08/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json
new file mode 100644
index 0000000..d2d059e
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_1.3_to_2.2.json
@@ -0,0 +1,948 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "1.3",
+ "target-version": "2.2",
+ "options": {
+ "config-types": {
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hbase-env": {
+ "merged-copy": "yes"
+ },
+ "oozie-env": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-env": {
+ "merged-copy": "yes"
+ },
+ "cluster-env" : {
+ "merged-copy": "yes"
+ },
+ "sqoop-env": {
+ "merged-copy": "yes"
+ },
+ "pig-env": {
+ "merged-copy": "yes"
+ },
+ "pig-properties": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "zookeeper-env": {
+ "merged-copy": "yes"
+ },
+ "hadoop-env": {
+ "merged-copy": "yes"
+ },
+ "hadoop-policy": {
+ "merged-copy": "yes"
+ },
+ "hbase-log4j": {
+ "merged-copy": "yes"
+ },
+ "hdfs-log4j": {
+ "merged-copy": "yes"
+ },
+ "hive-env": {
+ "merged-copy": "yes"
+ },
+ "hive-exec-log4j": {
+ "merged-copy": "yes"
+ },
+ "hive-log4j": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "oozie-site": {
+ "oozie.service.ProxyUserService.proxyuser.falcon.hosts": "*",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.service.SchemaService.wf.ext.schemas": "shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd",
+ "oozie.service.HadoopAccessorService.kerberos.enabled": "false",
+ "oozie.service.coord.push.check.requeue.interval": "30000",
+ "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials",
+ "oozie.authentication.kerberos.name.rules": "\n RULE:[2:$1@$0]([jt]t@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-MAPREDUSER/\n RULE:[2:$1@$0]([nd]n@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HDFSUSER/\n RULE:[2:$1@$0](hm@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\n RULE:[2:$1@$0](rs@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\n DEFAULT",
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.AuthorizationService.security.enabled": "true",
+ "oozie.services.ext": "org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService",
+ "oozie.services": "\n org.apache.oozie.service.SchedulerService,\n org.apache.oozie.service.InstrumentationService,\n org.apache.oozie.service.MemoryLocksService,\n org.apache.oozie.service.UUIDService,\n org.apache.oozie.service.ELService,\n org.apache.oozie.service.AuthorizationService,\n org.apache.oozie.service.UserGroupInformationService,\n org.apache.oozie.service.HadoopAccessorService,\n org.apache.oozie.service.JobsConcurrencyService,\n org.apache.oozie.service.URIHandlerService,\n org.apache.oozie.service.DagXLogInfoService,\n org.apache.oozie.service.SchemaService,\n org.apache.oozie.service.LiteWorkflowAppService,\n org.apache.oozie.service.JPAService,\n org.apache.oozie.service.StoreService,\n org.apache.oozie.service.CoordinatorStoreService,\n org.apache.oozie.service.SLAStoreService,\n org.apache.oozie.service.DBLiteWorkflowStoreService,\n org.apache.oozie.service.C
allbackService,\n org.apache.oozie.service.ShareLibService,\n org.apache.oozie.service.CallableQueueService,\n org.apache.oozie.service.ActionService,\n org.apache.oozie.service.ActionCheckerService,\n org.apache.oozie.service.RecoveryService,\n org.apache.oozie.service.PurgeService,\n org.apache.oozie.service.CoordinatorEngineService,\n org.apache.oozie.service.BundleEngineService,\n org.apache.oozie.service.DagEngineService,\n org.apache.oozie.service.CoordMaterializeTriggerService,\n org.apache.oozie.service.StatusTransitService,\n org.apache.oozie.service.PauseTransitService,\n org.apache.oozie.service.GroupsService,\n org.apache.oozie.service.ProxyUserService,\n org.apache.oozie.service.XLogStreamingService,\n org.apache.oozie.service.JvmPauseMonitorService",
+ "oozie.service.URIHandlerService.uri.handlers": "org.apache.oozie.dependency.FSURIHandler,org.apache.oozie.dependency.HCatURIHandler",
+ "oozie.service.ProxyUserService.proxyuser.falcon.groups": "*",
+ "oozie.service.ActionService.executor.ext.classes": "\n org.apache.oozie.action.email.EmailActionExecutor,\n org.apache.oozie.action.hadoop.HiveActionExecutor,\n org.apache.oozie.action.hadoop.ShellActionExecutor,\n org.apache.oozie.action.hadoop.SqoopActionExecutor,\n org.apache.oozie.action.hadoop.DistcpActionExecutor"
+ },
+ "hive-log4j": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Define some default values that can be overridden by system properties\nhive.log.threshold=ALL\nhive.root.logger=INFO,DRFA\nhive.log.dir=${java.io.tmpdir}/${user.name
}\nhive.log.file=hive.log\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hive.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshold=${hive.log.threshold}\n\n#\n# Daily Rolling File Appender\n#\n# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files\n# for different CLI session.\n#\n# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\n\nlog4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]:
%c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n\nlog4j.appender.console.encoding=UTF-8\n\n#custom logging levels\n#log4j.logger.xxx=DEBUG\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter\n\n\nlog4j.category.DataNucleus=ERROR,DRFA\nlog4j.category.Datastore=ERROR,DRFA\nlog4j.category.Datastore.Schema=ERROR,DRFA\nlog4j.category.JPOX.Datastore=ERROR,DRFA\nlog4j.category.JPOX.Plugin=ERROR,DRFA\nlog4j.category.JPOX.MetaData=ERROR,DRFA\nlog4j.category.JPOX.Query=ERROR,DRFA\nlog4j.category.JPOX.General=ERROR,DRFA\nlog4j.category.JPOX.Enha
ncer=ERROR,DRFA\n\n\n# Silence useless ZK logs\nlog4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA\nlog4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA"
+ },
+ "hive-exec-log4j": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Define some default values that can be overridden by system properties\n\nhive.log.threshold=ALL\nhive.root.logger=INFO,FA\nhive.log.dir=${java.io.tmpdir}/${user.name
}\nhive.query.id=hadoop\nhive.log.file=${hive.query.id}.log\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hive.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=${hive.log.threshold}\n\n#\n# File Appender\n#\n\nlog4j.appender.FA=org.apache.log4j.FileAppender\nlog4j.appender.FA.File=${hive.log.dir}/${hive.log.file}\nlog4j.appender.FA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\nlog4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:s
s} %p %c{2}: %m%n\n\n#custom logging levels\n#log4j.logger.xxx=DEBUG\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter\n\n\nlog4j.category.DataNucleus=ERROR,FA\nlog4j.category.Datastore=ERROR,FA\nlog4j.category.Datastore.Schema=ERROR,FA\nlog4j.category.JPOX.Datastore=ERROR,FA\nlog4j.category.JPOX.Plugin=ERROR,FA\nlog4j.category.JPOX.MetaData=ERROR,FA\nlog4j.category.JPOX.Query=ERROR,FA\nlog4j.category.JPOX.General=ERROR,FA\nlog4j.category.JPOX.Enhancer=ERROR,FA\n\n\n# Silence useless ZK logs\nlog4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA\nlog4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA"
+ },
+ "hive-env": {
+ "content": "\n if [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\nexport HADOOP_HEAPSIZE=\"{{hive_heapsize}}\"\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory
can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra libraries required for hive compilation/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\nelif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog\nfi\n\nexport METASTORE_PORT={{hive_metastore_port}}",
+ "hive_aux_jars_path": {"remove":"yes"},
+ "hive_conf_dir": {"remove":"yes"},
+ "hive_dbroot": {"remove":"yes"}
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.logg
er=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.c
onsole.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.security.logger
}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=fa
lse\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file
}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlo
g4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN"
+ },
+ "hbase-log4j": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# Define some default values that can be overridden by system properties\nhbase.root.logger=INFO,console\nhbase.security.logger=INFO,console\nhbase.log.dir=.\nhbase.l
og.file=hbase.log\n\n# Define the root logger to the system property \"hbase.root.logger\".\nlog4j.rootLogger=${hbase.root.logger}\n\n# Logging Threshold\nlog4j.threshold=ALL\n\n#\n# Daily Rolling File Appender\n#\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n# Rolling File Appender properties\nhbase.log.maxfilesize=256MB\nhbase.log.maxbackupindex=20\n\n# Rolling File Appender\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}\n\nlog4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}\nlog4j.appender.RFA.MaxBackupIndex=
${hbase.log.maxbackupindex}\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n#\n# Security audit appender\n#\nhbase.security.log.file=SecurityAuth.audit\nhbase.security.log.maxfilesize=256MB\nhbase.security.log.maxbackupindex=20\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}\nlog4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.category.SecurityLogger=${hbase.security.logger}\nlog4j.additivity.SecurityLogger=false\n#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE\n\n#\n# Null Appender\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\
n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n# Custom Logging levels\n\nlog4j.logger.org.apache.zookeeper=INFO\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.hbase=DEBUG\n# Make these two classes INFO-level. Make them DEBUG to see more zk debug.\nlog4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO\nlog4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO\n#log4j.logger.org.apache.hadoop.dfs=DEBUG\n# Set this class to log INFO only otherwise its OTT\n# Enable this to get detailed connection error/retry logging.\n# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE\n\n\n# Uncomment this line to enable tra
cing on _every_ RPC call (this can be a lot of output)\n#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG\n\n# Uncomment the below if you want to remove logging of client region caching'\n# and scan of .META. messages\n# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO\n# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO"
+ },
+ "hadoop-policy": {
+ "security.refresh.usertogroups.mappings.protocol.acl": "hadoop",
+ "security.refresh.policy.protocol.acl": "hadoop",
+ "security.admin.operations.protocol.acl": "hadoop",
+ "security.inter.datanode.protocol.acl": "*"
+ },
+ "hadoop-env": {
+ "namenode_formatted_mark_dir": {"remove": "yes"},
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appended
to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps
-Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The following a
pplies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HADOO
P_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /usr/share/ja
va/*mysql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n if [ -d \"/etc/tez/conf/\" ]; then\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n fi\nfi\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS
\""
+ },
+ "zookeeper-env": {
+ "content": "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}"
+ },
+ "webhcat-site": {
+ "templeton.hcat.home": "hive.tar.gz/hive/hcatalog",
+ "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+ "templeton.sqoop.home": "sqoop.tar.gz/sqoop",
+ "templeton.storage.class": "org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage",
+ "templeton.sqoop.archive": "hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop.tar.gz",
+ "templeton.hive.archive": "hdfs:///hdp/apps/${hdp.version}/hive/hive.tar.gz",
+ "templeton.streaming.jar": "hdfs:///hdp/apps/${hdp.version}/mapreduce/hadoop-streaming.jar",
+ "templeton.libjars": "/usr/hdp/current/zookeeper-client/zookeeper.jar",
+ "templeton.hadoop": "/usr/hdp/current/hadoop-client/bin/hadoop",
+ "templeton.hive.home": "hive.tar.gz/hive",
+ "templeton.sqoop.path": "sqoop.tar.gz/sqoop/bin/sqoop",
+ "templeton.hcat": "/usr/hdp/current/hive-client/bin/hcat",
+ "templeton.pig.archive": "hdfs:///hdp/apps/${hdp.version}/pig/pig.tar.gz"
+ },
+ "pig-properties": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.\n# see bin/pig -help\n\n# brief logging (no timestamps
)\nbrief=false\n\n# debug level, INFO is default\ndebug=INFO\n\n# verbose print all log messages to screen (default to print only INFO and above to screen)\nverbose=false\n\n# exectype local|mapreduce, mapreduce is default\nexectype=mapreduce\n\n# Enable insertion of information about script into hadoop job conf \npig.script.info.enabled=true\n\n# Do not spill temp files smaller than this size (bytes)\npig.spill.size.threshold=5000000\n\n# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)\n# This should help reduce the number of files being spilled.\npig.spill.gc.activation.size=40000000\n\n# the following two parameters are to help estimate the reducer number\npig.exec.reducers.bytes.per.reducer=1000000000\npig.exec.reducers.max=999\n\n# Temporary location to store the intermediate data.\npig.temp.dir=/tmp/\n\n# Threshold for merging FRJoin fragment files\npig.files.concatenation.threshold=100\npig.optimistic.files.concatenation=false;\n\npi
g.disable.counter=false\n\n# Avoid pig failures when multiple jobs write to the same location\npig.location.check.strict=false\n\nhcat.bin=/usr/bin/hcat"
+ },
+ "pig-env" : {
+ "content": "\nJAVA_HOME={{java64_home}}\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\nif [ -d \"/usr/lib/tez\" ]; then\n PIG_OPTS=\"$PIG_OPTS -Dmapreduce.framework.name=yarn\"\nfi"
+ },
+ "sqoop-env": {
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n#Set path to where bin/hadoop is available\n#Set path to where bin/hadoop is available\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n#set the path to where bin/hbase is available\nexport HBASE_HOME=${HBASE_HOME:-{{hbase_home}}}\n\n#Set the path to where bin/hive is available\nexport HIVE_HOME=${HIVE_HOME:-{{hive_home}}}\n\n#Set the path for where zookeper config dir is\nexport ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}\n\n# add libthrift in hive to sqoop class path first so hive imports work\nexport SQOOP_USER_CLASSPATH=\"`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}\""
+ },
+ "cluster-env": {
+ "ignore_groupsusers_create": "false"
+ },
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2",
+ "yarn.scheduler.capacity.maximum-applications": "10000",
+ "yarn.scheduler.capacity.root.acl_administer_queue": "*",
+ "yarn.scheduler.capacity.root.capacity": "100",
+ "yarn.scheduler.capacity.root.default.acl_administer_jobs": "*",
+ "yarn.scheduler.capacity.root.default.acl_submit_applications": "*",
+ "yarn.scheduler.capacity.root.default.capacity": "100",
+ "yarn.scheduler.capacity.root.default.maximum-capacity": "100",
+ "yarn.scheduler.capacity.root.default.state": "RUNNING",
+ "yarn.scheduler.capacity.root.default.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.queues": "default",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+ "yarn.scheduler.capacity.default.minimum-user-limit-percent": "100",
+ "yarn.scheduler.capacity.node-locality-delay": "40",
+ "yarn.scheduler.capacity.root.default.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+ "yarn.scheduler.capacity.root.default-node-label-expression": " "
+ },
+ "hbase-env": {
+ "content": "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to enable JMX
exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{log_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER
\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{re
gionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}"
+ },
+ "mapred-site": {
+ "hadoop.job.history.location": {"remove": "yes"},
+ "hadoop.job.history.user.location": {"remove": "yes"},
+ "io.sort.record.percent": {"remove": "yes"},
+ "jetty.connector": {"remove": "yes"},
+ "mapred.child.java.opts": {"remove": "yes"},
+ "mapred.child.root.logger": {"remove": "yes"},
+ "mapred.create.symlink": {"remove": "yes"},
+ "mapred.fairscheduler.allocation.file": {"remove": "yes"},
+ "mapred.fairscheduler.assignmultiple": {"remove": "yes"},
+ "mapreduce.job.priority": {"remove": "yes"},
+ "mapred.jobtracker.blacklist.fault-bucket-width": {"remove": "yes"},
+ "mapred.jobtracker.blacklist.fault-timeout-window": {"remove": "yes"},
+ "mapred.jobtracker.completeuserjobs.maximum": {"remove": "yes"},
+ "mapred.jobtracker.job.history.block.size": {"remove": "yes"},
+ "mapred.jobtracker.retirejob.check": {"remove": "yes"},
+ "mapred.jobtracker.retirejob.interval": {"remove": "yes"},
+ "mapred.jobtracker.taskScheduler": {"remove": "yes"},
+ "mapred.permissions.supergroup": {"remove": "yes"},
+ "mapred.queue.names": {"remove": "yes"},
+ "mapreduce.cluster.acls.enabled": {"remove": "yes"},
+ "mapreduce.cluster.local.dir": {"remove": "yes"},
+ "mapreduce.cluster.mapmemory.mb": {"remove": "yes"},
+ "mapreduce.cluster.permissions.supergroup": {"remove": "yes"},
+ "mapreduce.cluster.reducememory.mb": {"remove": "yes"},
+ "mapreduce.cluster.temp.dir": {"remove": "yes"},
+ "mapreduce.jobtracker.jobinit.threads": {"remove": "yes"},
+ "mapreduce.jobtracker.permissions.supergroup": {"remove": "yes"},
+ "mapreduce.job.cache.symlink.create": {"remove": "yes"},
+ "mapreduce.job.speculative.slownodethreshold": {"remove": "yes"},
+ "mapreduce.job.userlog.retain.hours": {"remove": "yes"},
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "mapreduce.am.max-attempts": "2",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.fileoutputcommitter.marksuccessfuljobs": {"remove": "yes"},
+ "mapreduce.framework.name": "yarn",
+ "mapreduce.history.server.embedded": {"remove": "yes"},
+ "mapreduce.history.server.http.address": {"remove": "yes"},
+ "apreduce.job.committer.setup.cleanup.needed": {"remove": "yes"},
+ "mapreduce.job.jvm.numtasks": {"remove": "yes"},
+ "mapreduce.jobhistory.address": {
+ "value": "{JOBHISTORY_HOST}:10020",
+ "template": "yes"
+ },
+ "mapreduce.jobhistory.done-dir": "/mr-history/done",
+ "mapreduce.jobhistory.intermediate-done-dir": "/mr-history/tmp",
+ "mapreduce.jobhistory.webapp.address": {
+ "value": "{JOBHISTORY_HOST}:19888",
+ "template": "yes"
+ },
+ "mapreduce.jobtracker.address": {"remove": "yes"},
+ "mapreduce.jobtracker.blacklist.average.threshold": {"remove": "yes"},
+ "mapreduce.jobtracker.expire.trackers.interval": {"remove": "yes"},
+ "mapreduce.jobtracker.handler.count": {"remove": "yes"},
+ "mapreduce.jobtracker.heartbeats.in.second": {"remove": "yes"},
+ "mapreduce.jobtracker.hosts.exclude.filename": {"remove": "yes"},
+ "mapreduce.jobtracker.hosts.filename": {"remove": "yes"},
+ "mapreduce.jobtracker.http.address": {"remove": "yes"},
+ "mapreduce.jobtracker.instrumentation": {"remove": "yes"},
+ "mapreduce.jobtracker.jobhistory.block.size": {"remove": "yes"},
+ "mapreduce.jobtracker.jobhistory.location": {"remove": "yes"},
+ "mapreduce.jobtracker.jobhistory.lru.cache.size": {"remove": "yes"},
+ "mapreduce.jobtracker.maxmapmemory.mb": {"remove": "yes"},
+ "mapreduce.jobtracker.maxreducememory.mb": {"remove": "yes"},
+ "mapreduce.jobtracker.maxtasks.perjob": {"remove": "yes"},
+ "mapreduce.jobtracker.persist.jobstatus.active": {"remove": "yes"},
+ "mapreduce.jobtracker.persist.jobstatus.dir": {"remove": "yes"},
+ "mapreduce.jobtracker.persist.jobstatus.hours": {"remove": "yes"},
+ "mapreduce.jobtracker.restart.recover": {"remove": "yes"},
+ "mapreduce.jobtracker.retiredjobs.cache.size": {"remove": "yes"},
+ "mapreduce.jobtracker.retirejobs": {"remove": "yes"},
+ "mapreduce.jobtracker.split.metainfo.maxsize": {"remove": "yes"},
+ "mapreduce.jobtracker.staging.root.dir": {"remove": "yes"},
+ "mapreduce.jobtracker.system.dir": {"remove": "yes"},
+ "mapreduce.jobtracker.taskcache.levels": {"remove": "yes"},
+ "mapreduce.jobtracker.taskscheduler": {"remove": "yes"},
+ "mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob": {"remove": "yes"},
+ "mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad": {"remove": "yes"},
+ "mapreduce.jobtracker.tasktracker.maxblacklists": {"remove": "yes"},
+ "mapreduce.jobtracker.webinterface.trusted": {"remove": "yes"},
+ "mapreduce.map.java.opts": "-Xmx546m",
+ "mapreduce.map.log.level": "INFO",
+ "mapreduce.map.memory.mb": "1024",
+ "mapreduce.map.output.compress": "false",
+ "mapreduce.map.output.compress.codec": {"remove": "yes"},
+ "mapreduce.map.sort.spill.percent": "0.7",
+ "mapreduce.output.fileoutputformat.compress": "false",
+ "mapreduce.reduce.input.limit": {"remove": "yes"},
+ "mapreduce.reduce.java.opts": "-Xmx756m",
+ "mapreduce.reduce.log.level": "INFO",
+ "mapreduce.reduce.memory.mb": "1024",
+ "mapreduce.reduce.merge.inmem.threshold": {"remove": "yes"},
+ "mapreduce.shuffle.port": "13562",
+ "mapreduce.task.timeout": "300000",
+ "mapreduce.task.userlog.limit.kb": {"remove": "yes"},
+ "mapreduce.tasktracker.cache.local.size": {"remove": "yes"},
+ "mapreduce.tasktracker.contention.tracking": {"remove": "yes"},
+ "mapreduce.tasktracker.dns.interface": {"remove": "yes"},
+ "mapreduce.tasktracker.dns.nameserver": {"remove": "yes"},
+ "mapreduce.tasktracker.events.batchsize": {"remove": "yes"},
+ "mapreduce.tasktracker.group": {"remove": "yes"},
+ "mapreduce.tasktracker.healthchecker.interval": {"remove": "yes"},
+ "mapreduce.tasktracker.healthchecker.script.args": {"remove": "yes"},
+ "mapreduce.tasktracker.healthchecker.script.path": {"remove": "yes"},
+ "mapreduce.tasktracker.healthchecker.script.timeout": {"remove": "yes"},
+ "mapreduce.tasktracker.host.name": {"remove": "yes"},
+ "mapreduce.tasktracker.http.address": {"remove": "yes"},
+ "mapreduce.tasktracker.http.threads": {"remove": "yes"},
+ "mapreduce.tasktracker.indexcache.mb": {"remove": "yes"},
+ "mapreduce.tasktracker.instrumentation": {"remove": "yes"},
+ "mapreduce.tasktracker.local.dir.minspacekill": {"remove": "yes"},
+ "mapreduce.tasktracker.local.dir.minspacestart": {"remove": "yes"},
+ "mapreduce.tasktracker.map.tasks.maximum": {"remove": "yes"},
+ "mapreduce.tasktracker.net.static.resolutions": {"remove": "yes"},
+ "mapreduce.tasktracker.reduce.tasks.maximum": {"remove": "yes"},
+ "mapreduce.tasktracker.report.address": {"remove": "yes"},
+ "mapreduce.tasktracker.resourcecalculatorplugin": {"remove": "yes"},
+ "mapreduce.tasktracker.taskcontroller": {"remove": "yes"},
+ "mapreduce.tasktracker.taskmemorymanager.monitoringinterval": {"remove": "yes"},
+ "mapreduce.tasktracker.tasks.sleeptimebeforesigkill": {"remove": "yes"},
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx312m -Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.log.level": "INFO",
+ "yarn.app.mapreduce.am.resource.mb": "512",
+ "yarn.app.mapreduce.am.staging-dir": "/user",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0"
+ },
+ "yarn-env": {
+ "apptimelineserver_heapsize": "1024",
+ "content": "\nexport HADOOP_YARN_HOME={{hadoop_yarn_home}}\nexport YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER\nexport YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\nexport JAVA_HOME={{java64_home}}\n\n# User for YARN daemons\nexport HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}\n\n# resolve links - $0 may be a softlink\nexport YARN_CONF_DIR=\"${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}\"\n\n# some Java parameters\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\nif [ \"$JAVA_HOME\" != \"\" ]; then\n #echo \"run java in $JAVA_HOME\"\n JAVA_HOME=$JAVA_HOME\nfi\n\nif [ \"$JAVA_HOME\" = \"\" ]; then\n echo \"Error: JAVA_HOME is not set.\"\n exit 1\nfi\n\nJAVA=$JAVA_HOME/bin/java\nJAVA_HEAP_MAX=-Xmx1000m\n\n# For setting YARN specific HEAP sizes please use this\n# Parameter and set appropriately\nYARN_HEAPSIZE={{yarn_heapsize}}\n\n# check envvars which might override default args\nif [ \"$YARN_HEAPSIZE\" != \"\" ]; then\n JAVA_HEAP_MAX
=\"-Xmx\"\"$YARN_HEAPSIZE\"\"m\"\nfi\n\n# Resource Manager specific parameters\n\n# Specify the max Heapsize for the ResourceManager using a numerical value\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n# the value to 1000.\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\n# and/or YARN_RESOURCEMANAGER_OPTS.\n# If not specified, the default value will be picked from either YARN_HEAPMAX\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\nexport YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}\n\n# Specify the JVM options to be used when starting the ResourceManager.\n# These options will be appended to the options specified as YARN_OPTS\n# and therefore may override any similar flags set in YARN_OPTS\n#export YARN_RESOURCEMANAGER_OPTS=\n\n# Node Manager specific parameters\n\n# Specify the max Heapsize for the NodeManager using a numerical value\n# in the scale of MB. For example, to sp
ecify an jvm option of -Xmx1000m, set\n# the value to 1000.\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\n# and/or YARN_NODEMANAGER_OPTS.\n# If not specified, the default value will be picked from either YARN_HEAPMAX\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\nexport YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}\n\n# Specify the max Heapsize for the HistoryManager using a numerical value\n# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n# the value to 1024.\n# This value will be overridden by an Xmx setting specified in either YARN_OPTS\n# and/or YARN_HISTORYSERVER_OPTS.\n# If not specified, the default value will be picked from either YARN_HEAPMAX\n# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\nexport YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}}\n\n# Specify the JVM options to be used when starting the NodeManager.\n# These options will be
appended to the options specified as YARN_OPTS\n# and therefore may override any similar flags set in YARN_OPTS\n#export YARN_NODEMANAGER_OPTS=\n\n# so that filenames w/ spaces are handled correctly in loops below\nIFS=\n\n\n# default log directory and file\nif [ \"$YARN_LOG_DIR\" = \"\" ]; then\n YARN_LOG_DIR=\"$HADOOP_YARN_HOME/logs\"\nfi\nif [ \"$YARN_LOGFILE\" = \"\" ]; then\n YARN_LOGFILE='yarn.log'\nfi\n\n# default policy file for service-level authorization\nif [ \"$YARN_POLICYFILE\" = \"\" ]; then\n YARN_POLICYFILE=\"hadoop-policy.xml\"\nfi\n\n# restore ordinary behaviour\nunset IFS\n\n\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR\"\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING\"\nYARN_OPTS=\"$YARN_OPTS -Dhadoop.root.logger=${
YARN_ROOT_LOGGER:-INFO,console}\"\nYARN_OPTS=\"$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\nif [ \"x$JAVA_LIBRARY_PATH\" != \"x\" ]; then\n YARN_OPTS=\"$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH\"\nfi\nYARN_OPTS=\"$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE\"",
+ "min_user_id": "1000",
+ "nodemanager_heapsize": "1024",
+ "resourcemanager_heapsize": "1024",
+ "yarn_heapsize": "1024",
+ "yarn_log_dir_prefix": "/var/log/hadoop-yarn",
+ "yarn_pid_dir_prefix": "/var/run/hadoop-yarn",
+ "yarn_user": "yarn"
+ },
+ "oozie-env": {
+ "content": "\n#!/bin/bash\n\nif [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n #export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\nfi\n\n#Set JAVA HOME\nexport JAVA_HOME={{java_home}}\n\nexport JRE_HOME=${JAVA_HOME}\n\n# Set Oozie specific environment variables here.\n\n# Settings for the Embedded Tomcat that runs Oozie\n# Java System properties for Oozie should be specified in this variable\n#\n# export CATALINA_OPTS=\n\n# Oozie configuration file to load from Oozie configuration directory\n#\n# export OOZIE_CONFIG_FILE=oozie-site.xml\n\n# Oozie logs directory\n#\nexport OOZIE_LOG={{oozie_log_dir}}\n\n# Oozie pid directory\n#\nexport CATALINA_PID={{pid_file}}\n\n#Location of the data for oozie\nexport OOZIE_DATA={{oozie_data_dir}}\n\n# Oozie Log4J configuration file to load from Oozie configuration
directory\n#\n# export OOZIE_LOG4J_FILE=oozie-log4j.properties\n\n# Reload interval of the Log4J configuration file, in seconds\n#\n# export OOZIE_LOG4J_RELOAD=10\n\n# The port Oozie server runs\n#\nexport OOZIE_HTTP_PORT={{oozie_server_port}}\n\n# The admin port Oozie server runs\n#\nexport OOZIE_ADMIN_PORT={{oozie_server_admin_port}}\n\n# The host name Oozie server runs on\n#\n# export OOZIE_HTTP_HOSTNAME=`hostname -f`\n\n# The base URL for callback URLs to Oozie\n#\n# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\nexport JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64\n\n# At least 1 minute of retry time to account for server downtime during\n# upgrade/downgrade\nexport OOZIE_CLIENT_OPTS=\"${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 \"\n\n# This is needed so that Oozie does not run into OOM or GC Overhead limit\n# exceeded exceptions. If the oozie server is handling large number of\n# workflows/coordinator jobs, the memory
settings may need to be revised\nexport CATALINA_OPTS=\"${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m \"",
+ "oozie_heapsize": "2048m",
+ "oozie_permsize": "256m",
+ "oozie_ambari_database": "MySQL",
+ "oozie_existing_mysql_host": "",
+ "oozie_existing_oracle_host": "",
+ "oozie_existing_postgresql_host": ""
+ },
+ "yarn-log4j":{
+ "content": "\n#Relative to Yarn Log Dir Prefix\nyarn.log.dir=.\n#\n# Job Summary Appender\n#\n# Use following logger to send summary to separate file defined by\n# hadoop.mapreduce.jobsummary.log.file rolled daily:\n# hadoop.mapreduce.jobsummary.logger=INFO,JSA\n#\nhadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}\nhadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log\nlog4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender\n# Set the ResourceManager summary log filename\nyarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log\n# Set the ResourceManager summary log level and appender\nyarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}\n#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\n\n# To enable AppSummaryLogging for the RM,\n# set yarn.server.resourcemanager.appsummary.logger to\n# LEVEL,RMSUMMARY in hadoop-env.sh\n\n# Appender for ResourceManager Application Summary Log\n# Requires the fo
llowing properties to be set\n# - hadoop.log.dir (Hadoop Log directory)\n# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)\n# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)\nlog4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender\nlog4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}\nlog4j.appender.RMSUMMARY.MaxFileSize=256MB\nlog4j.appender.RMSUMMARY.MaxBackupIndex=20\nlog4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.JSA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\nlog4j.appender.JSA.DatePattern=.yyyy-MM-dd\nlog4j.appender.JSA.layout=org.apache.log4j.PatternLayout\nlog4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$Applicat
ionSummary=${yarn.server.resourcemanager.appsummary.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false"
+ },
+ "yarn-site": {
+ "hadoop.registry.rm.enabled": "false",
+ "hadoop.registry.zk.quorum": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ },
+ "yarn.acl.enable": "false",
+ "yarn.admin.acl": "",
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "yarn.client.nodemanager-connect.max-wait-ms": "900000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.log-aggregation-enable": "true",
+ "yarn.log-aggregation.retain-seconds": "2592000",
+ "yarn.log.server.url": {
+ "value": "http://{JOBHISTORY_HOST}:19888/jobhistory/logs",
+ "template": "yes"
+ },
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.address": "0.0.0.0:45454",
+ "yarn.nodemanager.admin-env": "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX",
+ "yarn.nodemanager.aux-services": "mapreduce_shuffle",
+ "yarn.nodemanager.aux-services.mapreduce_shuffle.class": "org.apache.hadoop.mapred.ShuffleHandler",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
+ "yarn.nodemanager.container-monitor.interval-ms": "3000",
+ "yarn.nodemanager.delete.debug-delay-sec": "0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.disk-health-checker.min-healthy-disks": "0.25",
+ "yarn.nodemanager.health-checker.interval-ms": "135000",
+ "yarn.nodemanager.health-checker.script.timeout-ms": "60000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.group": "hadoop",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.local-dirs": "/hadoop/yarn/local",
+ "yarn.nodemanager.log-aggregation.compression-type": "gz",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-per-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.log-dirs": "/hadoop/yarn/log",
+ "yarn.nodemanager.log.retain-second": "604800",
+ "yarn.nodemanager.recovery.dir": "/var/log/hadoop-yarn/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "false",
+ "yarn.nodemanager.remote-app-log-dir": "/app-logs",
+ "yarn.nodemanager.remote-app-log-dir-suffix": "logs",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.memory-mb": "2048",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.nodemanager.vmem-check-enabled": "false",
+ "yarn.nodemanager.vmem-pmem-ratio": "2.1",
+ "yarn.resourcemanager.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8050",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.admin.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8141",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.am.max-attempts": "2",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.hostname": {
+ "value": "{RESOURCEMANAGER_HOST}",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.nodes.exclude-path": "/etc/hadoop/conf/yarn.exclude",
+ "yarn.resourcemanager.recovery.enabled": "false",
+ "yarn.resourcemanager.resource-tracker.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8025",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.scheduler.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8030",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.scheduler.class": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8088",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-address": "localhost:2181",
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.scheduler.maximum-allocation-mb": "2048",
+ "yarn.scheduler.minimum-allocation-mb": "682",
+ "yarn.timeline-service.address": {
+ "value": "{RESOURCEMANAGER_HOST}:10200",
+ "template": "yes"
+ },
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.generic-application-history.store-class": "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.path": "/hadoop/yarn/timeline",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms": "300000",
+ "yarn.timeline-service.store-class": "org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore",
+ "yarn.timeline-service.ttl-enable": "true",
+ "yarn.timeline-service.ttl-ms": "2678400000",
+ "yarn.timeline-service.webapp.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8188",
+ "template": "yes"
+ },
+ "yarn.timeline-service.webapp.https.address": {
+ "value": "{RESOURCEMANAGER_HOST}:8190",
+ "template": "yes"
+ }
+ },
+ "hdfs-site": {
+ "dfs.block.local-path-access.user": {"remove": "yes"},
+ "dfs.client.read.shortcircuit": "true",
+ "dfs.client.read.shortcircuit.streams.cache.size": "4096",
+ "dfs.datanode.du.pct": {"remove": "yes"},
+ "dfs.datanode.du.reserved": "1073741824",
+ "dfs.datanode.socket.write.timeout": {"remove": "yes"},
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "dfs.hosts": {"remove": "yes"},
+ "dfs.journalnode.http-address": "0.0.0.0:8480",
+ "dfs.secondary.https.port": {"remove": "yes"},
+ "dfs.web.ugi": {"remove": "yes"},
+ "fs.permissions.umask-mode": "022",
+ "ipc.server.max.response.size": {"remove": "yes"},
+ "ipc.server.read.threadpool.size": {"remove": "yes"},
+ "dfs.support.append": "true",
+ "dfs.namenode.checkpoint.txns": "1000000",
+ "dfs.namenode.checkpoint.period": "21600",
+ "dfs.namenode.checkpoint.edits.dir": "${dfs.namenode.checkpoint.dir}",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.http.policy": "HTTP_ONLY",
+ "dfs.namenode.name.dir.restore": "true",
+ "dfs.datanode.max.transfer.threads": "16384",
+ "dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
+ "dfs.namenode.checkpoint.period": "21600"
+ },
+ "core-site": {
+ "fs.checkpoint.size": {"remove": "yes"},
+ "hadoop.security.auth_to_local": "\n RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\n RULE:[2:$1@$0](jhs@.*)s/.*/mapred/\n RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\n RULE:[2:$1@$0](hm@.*)s/.*/hbase/\n RULE:[2:$1@$0](rs@.*)s/.*/hbase/\n DEFAULT\n ",
+ "hadoop.security.authentication": "simple",
+ "hadoop.security.authorization": "false",
+ "io.compression.codec.lzo.class": {"remove": "yes"},
+ "io.compression.codecs": "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec"
+ },
+ "mapred-env": {
+ "jtnode_heapsize": {"remove": "yes"},
+ "jtnode_opt_maxnewsize": {"remove": "yes"},
+ "jtnode_opt_newsize": {"remove": "yes"},
+ "lzo_enabled": {"remove": "yes"},
+ "rca_enabled": {"remove": "yes"},
+ "rca_properties": {"remove": "yes"},
+ "snappy_enabled": {"remove": "yes"},
+ "ttnode_heapsize": {"remove": "yes"},
+ "content": "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n#export HADOOP_JOB_HISTORYSERVER_OPTS=\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"",
+ "jobhistory_heapsize": "900",
+ "mapred_log_dir_prefix": "/var/log/hadoop-mapreduce",
+ "mapred_pid_dir_prefix": "/var/run/hadoop-mapreduce",
+ "mapred_user": "mapred"
+ },
+ "hive-site": {
+ "fs.hdfs.impl.disable.cache": {"remove":"yes"},
+ "fs.file.impl.disable.cache": {"remove":"yes"},
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authorization.manager": {"remove":"yes"},
+ "hive.security.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.fetch.task.aggr": "false",
+ "hive.execution.engine": "mr",
+ "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
+ "hive.vectorized.groupby.maxentries": "100000",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.compute.query.using.stats": "true",
+ "hive.stats.dbclass": "fs",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.exec.post.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "hive.cbo.enable": "true",
+ "hive.optimize.index.filter": "true",
+ "hive.optimize.bucketmapjoin": "true",
+ "hive.mapjoin.bucket.cache.size": "10000",
+ "hive.limit.optimize.enable": "true",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.metastore.sasl.enabled": "false",
+ "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager",
+ "hive.optimize.constant.propagation": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.metastore.kerberos.principal": "hive/_HOST@EXAMPLE.COM",
+ "hive.txn.max.open.batch": "1000",
+ "hive.exec.compress.output": "false",
+ "hive.merge.size.per.task": "256000000",
+ "hive.heapsize": "1024",
+ "hive.security.authenticator.manager": "org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator",
+ "hive.merge.mapfiles": "true",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.server2.enable.doAs": "true",
+ "hive.metastore.pre.event.listeners": "org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener",
+ "hive.metastore.failure.retries": "24",
+ "hive.server2.enable.impersonation": "true",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.server2.transport.mode": "binary",
+ "hive.tez.container.size": "682",
+ "hive.zookeeper.client.port": "2181",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "javax.jdo.option.ConnectionPassword": "1",
+ "hive.exec.max.created.files": "100000",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.orc.splits.include.file.footer": "false",
+ "hive.exec.pre.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.exec.failure.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.optimize.reducededuplication.min.reducer": "4",
+ "hive.enforce.sortmergebucketmapjoin": "true",
+ "hive.auto.convert.join.noconditionaltask.size": "238026752",
+ "javax.jdo.option.ConnectionUserName": "hive",
+ "hive.tez.log.level": "INFO",
+ "hive.compactor.delta.num.threshold": "10",
+ "hive.exec.dynamic.partition": "true",
+ "hive.server2.authentication": "NONE",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.tez.input.format": "org.apache.hadoop.hive.ql.io.HiveInputFormat",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.support.concurrency": "false",
+ "hive.compactor.check.interval": "300L",
+ "hive.compactor.delta.pct.threshold": "0.1f",
+ "hive.auto.convert.sortmerge.join.noconditionaltask": "true",
+ "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.prewarm.numcontainers": "10",
+ "hive.vectorized.groupby.flush.percent": "0.1",
+ "hive.server2.authentication.spnego.principal": "/etc/security/keytabs/spnego.service.keytab",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.vectorized.groupby.maxentries": "100000",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.zookeeper.connectString": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ },
+ "hive.compactor.abortedtxn.threshold": "1000",
+ "hive.compactor.initiator.on": "false",
+ "hive.compactor.worker.threads": "0",
+ "hive.compactor.worker.timeout": "86400L",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.fetch.task.conversion": "more",
+ "hive.limit.pushdown.memory.usage": "0.04",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.metastore.kerberos.keytab.file": "/etc/security/keytabs/hive.service.keytab",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.reducededuplication": "true",
+ "hive.prewarm.enabled": "false",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.authentication.spnego.keytab": "HTTP/_HOST@EXAMPLE.COM",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.tez.default.queues": "default",
+ "hive.server2.tez.initialize.default.sessions": "false",
+ "hive.server2.tez.sessions.per.default.queue": "1",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.use.SSL": "false",
+ "hive.stats.autogather": "true",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.txn.timeout": "300",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.enabled": "true",
+ "hive.zookeeper.quorum": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ }
+ },
+ "hbase-site": {
+ "dfs.support.append": {"remove": "yes"},
+ "hbase.defaults.for.version.skip": "true",
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.max.filesize": "10737418240",
+ "hbase.hstore.blockingStoreFiles": "10",
+ "hbase.hstore.flush.retries.number": "120",
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "hbase.regionserver.handler.count": "60",
+ "hbase.rpc.engine": {"remove": "yes"},
+ "hfile.block.cache.size": "0.40",
+ "zookeeper.session.timeout": "30000",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "hbase.local.dir": "${hbase.tmp.dir}/local",
+ "hbase.hstore.flush.retries.number": {"remove":"true"}
+ },
+ "webhcat-env": {
+ "content": "\n# The file containing the running pid\nPID_FILE={{webhcat_pid_file}}\n\nTEMPLETON_LOG_DIR={{templeton_log_dir}}/\n\n\nWEBHCAT_LOG_DIR={{templeton_log_dir}}/\n\n# The console error log\nERROR_LOG={{templeton_log_dir}}/webhcat-console-error.log\n\n# The console log\nCONSOLE_LOG={{templeton_log_dir}}/webhcat-console.log\n\n#TEMPLETON_JAR=templeton_jar_name\n\n#HADOOP_PREFIX=hadoop_prefix\n\n#HCAT_PREFIX=hive_prefix\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nexport HADOOP_HOME={{hadoop_home}}"
+ }
+ },
+ "property-mapping": {
+ "create.empty.dir.if.nonexist": "mapreduce.jobcontrol.createdir.ifnotexist",
+ "dfs.access.time.precision": "dfs.namenode.accesstime.precision",
+ "dfs.backup.address": "dfs.namenode.backup.address",
+ "dfs.backup.http.address": "dfs.namenode.backup.http-address",
+ "dfs.balance.bandwidthPerSec": "dfs.datanode.balance.bandwidthPerSec",
+ "dfs.block.size": "dfs.blocksize",
+ "dfs.data.dir": "dfs.datanode.data.dir",
+ "dfs.datanode.max.xcievers": "dfs.datanode.max.transfer.threads",
+ "dfs.df.interval": "fs.df.interval",
+ "dfs.federation.nameservice.id": "dfs.nameservice.id",
+ "dfs.federation.nameservices": "dfs.nameservices",
+ "dfs.http.address": "dfs.namenode.http-address",
+ "dfs.https.address": "dfs.namenode.https-address",
+ "dfs.https.client.keystore.resource": "dfs.client.https.keystore.resource",
+ "dfs.https.need.client.auth": "dfs.client.https.need-auth",
+ "dfs.max.objects": "dfs.namenode.max.objects",
+ "dfs.max-repl-streams": "dfs.namenode.replication.max-streams",
+ "dfs.name.dir": "dfs.namenode.name.dir",
+ "dfs.name.dir.restore": "dfs.namenode.name.dir.restore",
+ "dfs.name.edits.dir": "dfs.namenode.edits.dir",
+ "dfs.permissions": "dfs.permissions.enabled",
+ "dfs.permissions.supergroup": "dfs.permissions.superusergroup",
+ "dfs.read.prefetch.size": "dfs.client.read.prefetch.size",
+ "dfs.replication.considerLoad": "dfs.namenode.replication.considerLoad",
+ "dfs.replication.interval": "dfs.namenode.replication.interval",
+ "dfs.replication.min": "dfs.namenode.replication.min",
+ "dfs.replication.pending.timeout.sec": "dfs.namenode.replication.pending.timeout-sec",
+ "dfs.safemode.extension": "dfs.namenode.safemode.extension",
+ "dfs.safemode.threshold.pct": "dfs.namenode.safemode.threshold-pct",
+ "dfs.secondary.http.address": "dfs.namenode.secondary.http-address",
+ "dfs.socket.timeout": "dfs.client.socket-timeout",
+ "dfs.umaskmode": "fs.permissions.umask-mode",
+ "dfs.write.packet.size": "dfs.client-write-packet-size",
+ "fs.checkpoint.dir": "dfs.namenode.checkpoint.dir",
+ "fs.checkpoint.edits.dir": "dfs.namenode.checkpoint.edits.dir",
+ "fs.checkpoint.period": "dfs.namenode.checkpoint.period",
+ "fs.default.name": "fs.defaultFS",
+ "hadoop.configured.node.mapping": "net.topology.configured.node.mapping",
+ "hadoop.job.history.location": "mapreduce.jobtracker.jobhistory.location",
+ "hadoop.native.lib": "io.native.lib.available",
+ "hadoop.net.static.resolutions": "mapreduce.tasktracker.net.static.resolutions",
+ "hadoop.pipes.command-file.keep": "mapreduce.pipes.commandfile.preserve",
+ "hadoop.pipes.executable.interpretor": "mapreduce.pipes.executable.interpretor",
+ "hadoop.pipes.executable": "mapreduce.pipes.executable",
+ "hadoop.pipes.java.mapper": "mapreduce.pipes.isjavamapper",
+ "hadoop.pipes.java.recordreader": "mapreduce.pipes.isjavarecordreader",
+ "hadoop.pipes.java.recordwriter": "mapreduce.pipes.isjavarecordwriter",
+ "hadoop.pipes.java.reducer": "mapreduce.pipes.isjavareducer",
+ "hadoop.pipes.partitioner": "mapreduce.pipes.partitioner",
+ "heartbeat.recheck.interval": "dfs.namenode.heartbeat.recheck-interval",
+ "io.bytes.per.checksum": "dfs.bytes-per-checksum",
+ "io.sort.factor": "mapreduce.task.io.sort.factor",
+ "io.sort.mb": "mapreduce.task.io.sort.mb",
+ "io.sort.spill.percent": "mapreduce.map.sort.spill.percent",
+ "jobclient.completion.poll.interval": "mapreduce.client.completion.pollinterval",
+ "jobclient.output.filter": "mapreduce.client.output.filter",
+ "jobclient.progress.monitor.poll.interval": "mapreduce.client.progressmonitor.pollinterval",
+ "job.end.notification.url": "mapreduce.job.end-notification.url",
+ "job.end.retry.attempts": "mapreduce.job.end-notification.retry.attempts",
+ "job.end.retry.interval": "mapreduce.job.end-notification.retry.interval",
+ "job.local.dir": "mapreduce.job.local.dir",
+ "keep.failed.task.files": "mapreduce.task.files.preserve.failedtasks",
+ "keep.task.files.pattern": "mapreduce.task.files.preserve.filepattern",
+ "key.value.separator.in.input.line": "mapreduce.input.keyvaluelinerecordreader.key.value.separator",
+ "local.cache.size": "mapreduce.tasktracker.cache.local.size",
+ "map.input.file": "mapreduce.map.input.file",
+ "map.input.length": "mapreduce.map.input.length",
+ "map.input.start": "mapreduce.map.input.start",
+ "map.output.key.field.separator": "mapreduce.map.output.key.field.separator",
+ "map.output.key.value.fields.spec": "mapreduce.fieldsel.map.output.key.value.fields.spec",
+ "mapred.acls.enabled": "mapreduce.cluster.acls.enabled",
+ "mapred.binary.partitioner.left.offset": "mapreduce.partition.binarypartitioner.left.offset",
+ "mapred.binary.partitioner.right.offset": "mapreduce.partition.binarypartitioner.right.offset",
+ "mapred.cache.archives": "mapreduce.job.cache.archives",
+ "mapred.cache.archives.timestamps": "mapreduce.job.cache.archives.timestamps",
+ "mapred.cache.files": "mapreduce.job.cache.files",
+ "mapred.cache.files.timestamps": "mapreduce.job.cache.files.timestamps",
+ "mapred.cache.localArchives": "mapreduce.job.cache.local.archives",
+ "mapred.cache.localFiles": "mapreduce.job.cache.local.files",
+ "mapred.child.tmp": "mapreduce.task.tmp.dir",
+ "mapred.cluster.average.blacklist.threshold": "mapreduce.jobtracker.blacklist.average.threshold",
+ "mapred.cluster.map.memory.mb": "mapreduce.cluster.mapmemory.mb",
+ "mapred.cluster.max.map.memory.mb": "mapreduce.jobtracker.maxmapmemory.mb",
+ "mapred.cluster.max.reduce.memory.mb": "mapreduce.jobtracker.maxreducememory.mb",
+ "mapred.cluster.reduce.memory.mb": "mapreduce.cluster.reducememory.mb",
+ "mapred.committer.job.setup.cleanup.needed": "mapreduce.job.committer.setup.cleanup.needed",
+ "mapred.compress.map.output": "mapreduce.map.output.compress",
+ "mapred.data.field.separator": "mapreduce.fieldsel.data.field.separator",
+ "mapred.debug.out.lines": "mapreduce.task.debugout.lines",
+ "mapred.healthChecker.interval": "mapreduce.tasktracker.healthchecker.interval",
+ "mapred.healthChecker.script.args": "mapreduce.tasktracker.healthchecker.script.args",
+ "mapred.healthChecker.script.path": "mapreduce.tasktracker.healthchecker.script.path",
+ "mapred.healthChecker.script.timeout": "mapreduce.tasktracker.hea
<TRUNCATED>
[32/55] [abbrv] ambari git commit: Updated team page (added Bob
Nettleton as PMC)
Posted by ol...@apache.org.
Updated team page (added Bob Nettleton as PMC)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/df1b77b6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/df1b77b6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/df1b77b6
Branch: refs/heads/branch-dev-logsearch
Commit: df1b77b620fc3acc2fe01188ff374c23450b703e
Parents: 48831ba
Author: Yusaku Sako <yu...@hortonworks.com>
Authored: Fri Aug 19 15:28:01 2016 -0700
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Fri Aug 19 15:28:01 2016 -0700
----------------------------------------------------------------------
docs/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/df1b77b6/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 4a7e888..82a49df 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -741,7 +741,7 @@
<email>rnettleton@apache.org</email>
<timezone>-8</timezone>
<roles>
- <role>Committer</role>
+ <role>PMC</role>
</roles>
<organization>
Hortonworks
[42/55] [abbrv] ambari git commit: AMBARI-18198. Doc updates about
ldap sync related properties (oleewere)
Posted by ol...@apache.org.
AMBARI-18198. Doc updates about ldap sync related properties (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/56fe3368
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/56fe3368
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/56fe3368
Branch: refs/heads/branch-dev-logsearch
Commit: 56fe3368bcfee0fb58b1bd1635d0285d1274667f
Parents: e74ecc9
Author: oleewere <ol...@gmail.com>
Authored: Thu Aug 18 17:22:50 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:37:31 2016 +0200
----------------------------------------------------------------------
ambari-server/docs/configuration/index.md | 8 +++----
.../server/configuration/Configuration.java | 24 ++++++++++++--------
2 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/56fe3368/ambari-server/docs/configuration/index.md
----------------------------------------------------------------------
diff --git a/ambari-server/docs/configuration/index.md b/ambari-server/docs/configuration/index.md
index 311def5..18234d0 100644
--- a/ambari-server/docs/configuration/index.md
+++ b/ambari-server/docs/configuration/index.md
@@ -85,10 +85,10 @@ The following are the properties which can be used to configure Ambari.
| authentication.ldap.primaryUrl | The LDAP URL used for connecting to an LDAP server when authenticating users. This should include both the host name and port. |`localhost:33389` |
| authentication.ldap.referral | Determines whether to follow LDAP referrals to other URLs when the LDAP controller doesn't have the requested object. |`follow` |
| authentication.ldap.secondaryUrl | A second LDAP URL to use as a backup when authenticating users. This should include both the host name and port. | |
-| authentication.ldap.sync.groupMemberFilter | The default filter to use for syncing member from LDAP. | |
-| authentication.ldap.sync.groupMemberReplacePattern | The default regex pattern to use when replacing the group member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID<br/><br/>The following are examples of valid values:<ul><li>``${member}``</ul> | |
-| authentication.ldap.sync.userMemberFilter | The default filter to use for syncing users from LDAP. | |
-| authentication.ldap.sync.userMemberReplacePattern | The default regex pattern to use when replacing the user member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID<br/><br/>The following are examples of valid values:<ul><li>``${member}``</ul> | |
+| authentication.ldap.sync.groupMemberFilter | Filter to use for syncing group members of a group from LDAP. (by default it is not used)<br/><br/>The following are examples of valid values:<ul><li>`(&(objectclass=posixgroup)(cn={member}))`</ul> | |
+| authentication.ldap.sync.groupMemberReplacePattern | Regex pattern to use when replacing the group member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID (e.g.: `member: <SID=123>;<GID=123>;cn=myCn,dc=org,dc=apache`)<br/><br/>The following are examples of valid values:<ul><li>`(?<sid>.*);(?<guid>.*);(?<member>.*)`</ul> | |
+| authentication.ldap.sync.userMemberFilter | Filter to use for syncing user members of a group from LDAP (by default it is not used).<br/><br/>The following are examples of valid values:<ul><li>`(&(objectclass=posixaccount)(uid={member}))`</ul> | |
+| authentication.ldap.sync.userMemberReplacePattern | Regex pattern to use when replacing the user member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID (e.g.: `member: <SID=123>;<GID=123>;cn=myCn,dc=org,dc=apache`)<br/><br/>The following are examples of valid values:<ul><li>`(?<sid>.*);(?<guid>.*);(?<member>.*)`</ul> | |
| authentication.ldap.useSSL | Determines whether to use LDAP over SSL (LDAPS). |`false` |
| authentication.ldap.userBase | The filter used when searching for users in LDAP. |`ou=people,dc=ambari,dc=apache,dc=org` |
| authentication.ldap.userObjectClass | The class to which user objects in LDAP belong. |`person` |
http://git-wip-us.apache.org/repos/asf/ambari/blob/56fe3368/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 8ab9091..3c37789 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -1055,41 +1055,45 @@ public class Configuration {
"authentication.ldap.pagination.enabled", "true");
/**
- * The default regex pattern to use when replacing the user member attribute
+ * Regex pattern to use when replacing the user member attribute
* ID value with a placeholder. This is used in cases where a UID of an LDAP
* member is not a full CN or unique ID.
*/
@Markdown(
- description = "The default regex pattern to use when replacing the user member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID",
- examples = { "`${member}`" })
+ description = "Regex pattern to use when replacing the user member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID (e.g.: `member: <SID=123>;<GID=123>;cn=myCn,dc=org,dc=apache`)",
+ examples = { "(?<sid>.*);(?<guid>.*);(?<member>.*)" })
public static final ConfigurationProperty<String> LDAP_SYNC_USER_MEMBER_REPLACE_PATTERN = new ConfigurationProperty<>(
"authentication.ldap.sync.userMemberReplacePattern",
LDAP_SYNC_MEMBER_REPLACE_PATTERN_DEFAULT);
/**
- * The default regex pattern to use when replacing the group member attribute
+ * Regex pattern to use when replacing the group member attribute
* ID value with a placeholder. This is used in cases where a UID of an LDAP
* member is not a full CN or unique ID.
*/
@Markdown(
- description = "The default regex pattern to use when replacing the group member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID",
- examples = { "`${member}`" })
+ description = "Regex pattern to use when replacing the group member attribute ID value with a placeholder. This is used in cases where a UID of an LDAP member is not a full CN or unique ID (e.g.: `member: <SID=123>;<GID=123>;cn=myCn,dc=org,dc=apache`)",
+ examples = { "(?<sid>.*);(?<guid>.*);(?<member>.*)" })
public static final ConfigurationProperty<String> LDAP_SYCN_GROUP_MEMBER_REPLACE_PATTERN = new ConfigurationProperty<>(
"authentication.ldap.sync.groupMemberReplacePattern",
LDAP_SYNC_MEMBER_REPLACE_PATTERN_DEFAULT);
/**
- * The default filter to use for syncing users from LDAP.
+ * Filter to use for syncing user members of group from LDAP. (by default it is not used)
*/
- @Markdown(description = "The default filter to use for syncing users from LDAP.")
+ @Markdown(
+ description = "Filter to use for syncing user members of a group from LDAP (by default it is not used).",
+ examples = {"(&(objectclass=posixaccount)(uid={member}))"})
public static final ConfigurationProperty<String> LDAP_SYNC_USER_MEMBER_FILTER = new ConfigurationProperty<>(
"authentication.ldap.sync.userMemberFilter",
LDAP_SYNC_MEMBER_FILTER_DEFAULT);
/**
- * The default filter to use for syncing member from LDAP.
+ * Filter to use for syncing group members of a group from LDAP. (by default it is not used)
*/
- @Markdown(description = "The default filter to use for syncing member from LDAP.")
+ @Markdown(
+ description = "Filter to use for syncing group members of a group from LDAP. (by default it is not used)",
+ examples = {"(&(objectclass=posixgroup)(cn={member}))"})
public static final ConfigurationProperty<String> LDAP_SYNC_GROUP_MEMBER_FILTER = new ConfigurationProperty<>(
"authentication.ldap.sync.groupMemberFilter",
LDAP_SYNC_MEMBER_FILTER_DEFAULT);
[48/55] [abbrv] ambari git commit: AMBARI-18077. Clean up Log Search
Appender and improve speed (Miklos Gergely via oleewere)
Posted by ol...@apache.org.
AMBARI-18077. Clean up Log Search Appender and improve speed (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f1f5ea64
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f1f5ea64
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f1f5ea64
Branch: refs/heads/branch-dev-logsearch
Commit: f1f5ea646dba73bb8dad658b19f800461dc1c389
Parents: 34b9e40
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Thu Aug 18 13:22:00 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:03 2016 +0200
----------------------------------------------------------------------
.../ambari-logsearch-appender/pom.xml | 1 -
.../logsearch/appender/LogsearchConversion.java | 57 ++++++------
.../ambari/logsearch/appender/Output.java | 91 ++++++++++++++++++++
.../apache/ambari/logsearch/appender/VBase.java | 63 --------------
.../ambari/logsearch/appender/VOutput.java | 88 -------------------
.../commands/AbstractStateFileZkCommand.java | 2 +-
6 files changed, 117 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f5ea64/ambari-logsearch/ambari-logsearch-appender/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/pom.xml b/ambari-logsearch/ambari-logsearch-appender/pom.xml
index 39f250a..e14d576 100644
--- a/ambari-logsearch/ambari-logsearch-appender/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-appender/pom.xml
@@ -67,7 +67,6 @@
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
- <version>1.2.17</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f5ea64/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
index dbdfe6c..877fa24 100644
--- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
+++ b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java
@@ -26,54 +26,47 @@ import org.apache.log4j.EnhancedPatternLayout;
import org.apache.log4j.spi.LoggingEvent;
public class LogsearchConversion extends EnhancedPatternLayout {
- //
- protected final int BUF_SIZE = 256;
- protected final int MAX_CAPACITY = 1024;
- private StringBuffer sbuf = new StringBuffer(BUF_SIZE);
-
- private String newLine = System.getProperty("line.separator");
+ private static final String NEW_LINE = System.getProperty("line.separator");
public LogsearchConversion() {
}
public String format(LoggingEvent event) {
- if (sbuf.capacity() > MAX_CAPACITY) {
- sbuf = new StringBuffer(BUF_SIZE);
- } else {
- sbuf.setLength(0);
- }
String outputStr = createOutput(event);
- sbuf.append(outputStr + newLine);
- return sbuf.toString();
+ return outputStr + NEW_LINE;
}
public String createOutput(LoggingEvent event) {
- VOutput vOutput = new VOutput();
- vOutput.setLevel(event.getLevel().toString());
- vOutput.setFile(event.getLocationInformation().getFileName());
- vOutput.setLine_number(Integer.parseInt(event.getLocationInformation().getLineNumber()));
- String logmsg = event.getMessage() != null ? event.getMessage().toString() : "";
+ Output output = new Output();
+
+ output.setLevel(event.getLevel().toString());
+ output.setFile(event.getLocationInformation().getFileName());
+ output.setLineNumber(Integer.parseInt(event.getLocationInformation().getLineNumber()));
+ output.setLogtime(Long.toString(event.getTimeStamp()));
+ output.setLoggerName(event.getLoggerName());
+ output.setThreadName(event.getThreadName());
+ output.setLogMessage(getLogMessage(event));
+
+ return output.toJson();
+ }
+
+ public String getLogMessage(LoggingEvent event) {
+ String logMessage = event.getMessage() != null ? event.getMessage().toString() : "";
+
if (event.getThrowableInformation() != null && event.getThrowableInformation().getThrowable() != null) {
- logmsg += newLine + stackTraceToString(event.getThrowableInformation().getThrowable());
+ logMessage += NEW_LINE;
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ event.getThrowableInformation().getThrowable().printStackTrace(pw);
+ logMessage += sw.toString();
}
- vOutput.setLog_message(logmsg);
- vOutput.setLogtime("" + event.getTimeStamp());
- vOutput.setLogger_name("" + event.getLoggerName());
- vOutput.setThread_name(event.getThreadName());
- return vOutput.toJson();
- }
- public String stackTraceToString(Throwable e) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- e.printStackTrace(pw);
- return sw.toString();
+ return logMessage;
}
-
+
@Override
public boolean ignoresThrowable() {
- //set false to ignore exception stacktrace
return false;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f5ea64/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/Output.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/Output.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/Output.java
new file mode 100644
index 0000000..8001054
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/Output.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.appender;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+import com.google.gson.stream.JsonWriter;
+
+class Output {
+
+ private String level;
+ private String file;
+ private String threadName;
+ private int lineNumber;
+ private String loggerName;
+ private String logtime;
+ private String logMessage;
+
+ void setLevel(String level) {
+ this.level = level;
+ }
+
+ void setFile(String file) {
+ this.file = file;
+ }
+
+ void setThreadName(String threadName) {
+ this.threadName = threadName;
+ }
+
+ void setLineNumber(int lineNumber) {
+ this.lineNumber = lineNumber;
+ }
+
+ void setLoggerName(String loggerName) {
+ this.loggerName = loggerName;
+ }
+
+ void setLogtime(String logtime) {
+ this.logtime = logtime;
+ }
+
+ void setLogMessage(String logMessage) {
+ this.logMessage = logMessage;
+ }
+
+ public String toJson() {
+ StringWriter stringWriter = new StringWriter();
+
+ try (JsonWriter writer = new JsonWriter(stringWriter)) {
+ writer.beginObject();
+
+ if (level != null) writer.name("level").value(level);
+ if (file != null) writer.name("file").value(file);
+ if (threadName != null) writer.name("thread_name").value(threadName);
+ writer.name("line_number").value(lineNumber);
+ if (loggerName != null) writer.name("logger_name").value(loggerName);
+ if (logtime != null) writer.name("logtime").value(logtime);
+ if (logMessage != null) writer.name("log_message").value(logMessage);
+
+ writer.endObject();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return stringWriter.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toJson();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f5ea64/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VBase.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VBase.java
deleted file mode 100644
index 4e91ccc..0000000
--- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VBase.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logsearch.appender;
-
-import java.lang.reflect.Field;
-
-import org.apache.log4j.Logger;
-
-import com.google.gson.Gson;
-
-public class VBase {
- private static Logger logger = Logger.getLogger(VBase.class);
-
- /**
- *
- */
- @Override
- public String toString() {
- @SuppressWarnings("rawtypes")
- Class klass = this.getClass();
- Field[] fields = klass.getDeclaredFields();
- StringBuilder builder = new StringBuilder(klass.getSimpleName() + "={");
- for (Field field : fields) {
- try {
- field.setAccessible(true);
- Object fieldValue = field.get(this);
- String fieldName = field.getName();
- if (!fieldName.equalsIgnoreCase("serialVersionUID")) {
- builder.append(fieldName + "={" + fieldValue + "} ");
- }
-
- } catch (Exception e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- }
- builder.append("}");
-
- return builder.toString();
- }
-
- public String toJson() {
- Gson gson = new Gson();
- String json = gson.toJson(this);
- return json;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f5ea64/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VOutput.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VOutput.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VOutput.java
deleted file mode 100644
index 61da1da..0000000
--- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/VOutput.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logsearch.appender;
-
-public class VOutput extends VBase {
-
- private String level;
- private String file;
- private String thread_name;
- private int line_number;
- private String log_message;
- private String logger_name;
- private String logtime;
-
- public String getLevel() {
- return level;
- }
-
- public void setLevel(String level) {
- this.level = level;
- }
-
- public String getFile() {
- return file;
- }
-
- public void setFile(String file) {
- this.file = file;
- }
-
- public String getThread_name() {
- return thread_name;
- }
-
- public void setThread_name(String thread_name) {
- this.thread_name = thread_name;
- }
-
- public int getLine_number() {
- return line_number;
- }
-
- public void setLine_number(int line_number) {
- this.line_number = line_number;
- }
-
- public String getLog_message() {
- return log_message;
- }
-
- public void setLog_message(String log_message) {
- this.log_message = log_message;
- }
-
- public String getLogger_name() {
- return logger_name;
- }
-
- public void setLogger_name(String logger_name) {
- this.logger_name = logger_name;
- }
-
- public String getLogtime() {
- return logtime;
- }
-
- public void setLogtime(String logtime) {
- this.logtime = logtime;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f1f5ea64/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java
index 664cd88..d351589 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/AbstractStateFileZkCommand.java
@@ -19,7 +19,7 @@
package org.apache.ambari.logsearch.solr.commands;
import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.domain.AmbariSolrState;;
+import org.apache.ambari.logsearch.solr.domain.AmbariSolrState;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
[14/55] [abbrv] ambari git commit: AMBARI-18202. On UI Sometimes:
after clicking delete property button,
property does not deleted instead Configuration Group window is opened
(alexantonenko)
Posted by ol...@apache.org.
AMBARI-18202. On UI Sometimes: after clicking delete property button, property does not deleted instead Configuration Group window is opened (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d3f8a4c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d3f8a4c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d3f8a4c
Branch: refs/heads/branch-dev-logsearch
Commit: 1d3f8a4ce7b847ca150c7276fc8a417488290e0a
Parents: 7888638
Author: Alex Antonenko <hi...@gmail.com>
Authored: Thu Aug 18 19:29:02 2016 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Aug 18 20:11:11 2016 +0300
----------------------------------------------------------------------
ambari-web/app/templates/common/configs/controls.hbs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d3f8a4c/ambari-web/app/templates/common/configs/controls.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/common/configs/controls.hbs b/ambari-web/app/templates/common/configs/controls.hbs
index ee67594..0a7d21e 100644
--- a/ambari-web/app/templates/common/configs/controls.hbs
+++ b/ambari-web/app/templates/common/configs/controls.hbs
@@ -42,7 +42,7 @@
</a>
{{/if}}
{{#if view.showRemove}}
- <a href="#" data-toggle="tooltip" class="btn-small"
+ <a href="#" data-toggle="tooltip" class="btn-small remove-config-property-button"
{{action "removeProperty" this target="view.parentView"}}
{{translateAttr data-original-title="common.remove"}}>
<i class="icon-minus-sign"></i>
[43/55] [abbrv] ambari git commit: AMBARI-18153.
Spelling/Capitalization Change for Log Search Smart Configuration (Dharmesh
Makwana via oleewere)
Posted by ol...@apache.org.
AMBARI-18153. Spelling/Capitalization Change for Log Search Smart Configuration (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2c7164dd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2c7164dd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2c7164dd
Branch: refs/heads/branch-dev-logsearch
Commit: 2c7164ddeee3e7440be3b8772b415040d6e82405
Parents: 74b1952
Author: oleewere <ol...@gmail.com>
Authored: Thu Aug 18 13:18:46 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:02 2016 +0200
----------------------------------------------------------------------
.../src/main/webapp/scripts/views/common/Header.js | 2 +-
.../LOGSEARCH/0.5.0/configuration/logsearch-properties.xml | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2c7164dd/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
index 753a7c3..cb8ca4e 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js
@@ -168,7 +168,7 @@ define(['require',
require(['views/filter/CreateLogfeederFilterView'],function(CreateLogfeederFilter){
var view = new CreateLogfeederFilter({});
var options = {
- title: "Logfeeder Filter",
+ title: "Log Feeder Log Levels",
content: view,
viewType: 'Filter',
resizable: false,
http://git-wip-us.apache.org/repos/asf/ambari/blob/2c7164dd/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
index 7f134e5..56ff2d1 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
@@ -115,8 +115,8 @@
<property>
<name>logsearch.logfeeder.include.default.level</name>
<value>FATAL,ERROR,WARN</value>
- <description>Include default Logfeeder log levels for Log Search. Used for bootstrapping the configuration only. (levels: FATAL,ERROR,WARN,INFO,DEBUG,TRACE)</description>
- <display-name>Logfeeder log levels</display-name>
+ <description>Include default Log Feeder Log Levels for Log Search. Used for bootstrapping the configuration only. (levels: FATAL,ERROR,WARN,INFO,DEBUG,TRACE)</description>
+ <display-name>Log Feeder Log Levels</display-name>
<value-attributes>
<editable-only-at-install>true</editable-only-at-install>
</value-attributes>
[04/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
new file mode 100644
index 0000000..6dace0f
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.4.json
@@ -0,0 +1,499 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.1",
+ "target-version": "2.2.4",
+ "options": {
+ "config-types": {
+ "capacity-scheduler": {
+ "merged-copy": "yes"
+ },
+ "cluster-env": {
+ "merged-copy": "yes"
+ },
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "falcon-startup.properties": {
+ "merged-copy": "yes"
+ },
+ "flume-env": {
+ "merged-copy": "yes"
+ },
+ "hadoop-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-log4j": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "hive-env": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-env": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "oozie-env": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "storm-env": {
+ "merged-copy": "yes"
+ },
+ "storm-site": {
+ "merged-copy": "yes"
+ },
+ "tez-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-log4j": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+ "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.default-node-label-expression": " "
+ },
+ "cluster-env": {
+ "smokeuser_principal_name": {"remove": "yes"}
+ },
+ "core-site": {
+ "hadoop.http.authentication.simple.anonymous.allowed": "true",
+ "hadoop.security.auth_to_local": "\n DEFAULT"
+ },
+ "falcon-startup.properties": {
+ "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService,\\\n org.apache.falcon.workflow.WorkflowJobEndNotificationService, \\\n org.apache.falcon.service.ProcessSubscriberService,\\\n org.apache.falcon.entity.store.ConfigurationStore,\\\n org.apache.falcon.rerun.service.RetryService,\\\n org.apache.falcon.rerun.service.LateRunService,\\\n org.apache.falcon.service.LogCleanupService,\\\n org.apache.falcon.metadata.MetadataMappingService",
+ "*.falcon.enableTLS": "false",
+ "*.falcon.graph.blueprints.graph": "com.thinkaurelius.titan.core.TitanFactory",
+ "*.falcon.graph.storage.backend": "berkeleyje",
+ "*.falcon.security.authorization.admin.groups": "falcon",
+ "*.falcon.security.authorization.admin.users": "falcon,ambari-qa",
+ "*.falcon.security.authorization.enabled": "false",
+ "*.falcon.security.authorization.provider": "org.apache.falcon.security.DefaultAuthorizationProvider",
+ "*.falcon.security.authorization.superusergroup": "falcon",
+ "*.journal.impl": "org.apache.falcon.transaction.SharedFileSystemJournal",
+ "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
+ "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap"
+ },
+ "flume-env": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced\n# during Flume startup.\n\n# Enviroment variables can be set here.\n\nexport JAVA_HOME={{java_home}}\
n\n# Give Flume more memory and pre-allocate, enable remote monitoring via JMX\n# export JAVA_OPTS=\"-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote\"\n\n# Note that the Flume conf directory is always included in the classpath.\n# Add flume sink to classpath\nif [ -e \"/usr/lib/flume/lib/ambari-metrics-flume-sink.jar\" ]; then\n export FLUME_CLASSPATH=$FLUME_CLASSPATH:/usr/lib/flume/lib/ambari-metrics-flume-sink.jar\nfi\n\nexport HIVE_HOME={{flume_hive_home}}\nexport HCAT_HOME={{flume_hcat_home}}"
+ },
+ "hadoop-env": {
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appended to HAD
OOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{
{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The following applies
to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HADOOP_MAST
ER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /usr/share/java/*my
sql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n if [ -d \"/etc/tez/conf/\" ]; then\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n fi\nfi\n\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"",
+ "namenode_opt_maxnewsize": "256m",
+ "namenode_opt_newsize": "256m"
+ },
+ "hbase-env": {
+ "content": "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to enable JMX expo
rting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{log_dir}}
\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport HBASE_REGIONSERVER_OPTS=
\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {"remove": "yes"}
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.logger=INF
O,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console
.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.security.logger}\nhad
oop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\nl
og4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n#
Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.ap
pender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN"
+ },
+ "hdfs-site": {
+ "dfs.datanode.max.transfer.threads": "16384",
+ "dfs.namenode.handler.count": "100",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600"
+ },
+ "hive-env": {
+ "content": "\n if [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra libraries required for hive compilat
ion/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n if [ -f \"${HIVE_AUX_JARS_PATH}\" ]; then \n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\n elif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\n fi\nelif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\nfi \n\nexport METASTORE_PORT={{hive_metastore_port}}",
+ "hive_dbroot": {"remove": "yes"}
+ },
+ "hive-site": {
+ "fs.file.impl.disable.cache": {"remove": "yes"},
+ "fs.hdfs.impl.disable.cache": {"remove": "yes"},
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.execution.engine": "tez",
+ "hive.heapsize": {"remove": "yes"},
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.mapjoin.mapreduce": {"remove": "yes"},
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.support.dynamic.service.discovery": "true",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "hive.auto.convert.sortmerge.join.noconditionaltask": {"remove": "yes"},
+ "hive.server2.enable.impersonation": {"remove": "yes"}
+ },
+ "mapred-env": {
+ "content": "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n#export HADOOP_JOB_HISTORYSERVER_OPTS=\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\""
+ },
+ "mapred-site": {
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "yarn.app.mapreduce.am.admin-command-opts": {"remove": "yes"},
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}"
+ },
+ "oozie-env": {
+ "content": "\n#!/bin/bash\n\nif [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\nfi\n\n#Set JAVA HOME\nexport JAVA_HOME={{java_home}}\n\nexport JRE_HOME=${JAVA_HOME}\n\n# Set Oozie specific environment variables here.\n\n# Settings for the Embedded Tomcat that runs Oozie\n# Java System properties for Oozie should be specified in this variable\n#\n# export CATALINA_OPTS=\n\n# Oozie configuration file to load from Oozie configuration directory\n#\n# export OOZIE_CONFIG_FILE=oozie-site.xml\n\n# Oozie logs directory\n#\nexport OOZIE_LOG={{oozie_log_dir}}\n\n# Oozie pid directory\n#\nexport CATALINA_PID={{pid_file}}\n\n#Location of the data for oozie\nexport OOZIE_DATA={{oozie_data_dir}}\n\n# Oozie Log4J configuration file to load from Oozie configuration directo
ry\n#\n# export OOZIE_LOG4J_FILE=oozie-log4j.properties\n\n# Reload interval of the Log4J configuration file, in seconds\n#\n# export OOZIE_LOG4J_RELOAD=10\n\n# The port Oozie server runs\n#\nexport OOZIE_HTTP_PORT={{oozie_server_port}}\n\n# The admin port Oozie server runs\n#\nexport OOZIE_ADMIN_PORT={{oozie_server_admin_port}}\n\n# The host name Oozie server runs on\n#\n# export OOZIE_HTTP_HOSTNAME=`hostname -f`\n\n# The base URL for callback URLs to Oozie\n#\n# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\nexport JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64\n\n# At least 1 minute of retry time to account for server downtime during\n# upgrade/downgrade\nexport OOZIE_CLIENT_OPTS=\"${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 \"\n\n# This is needed so that Oozie does not run into OOM or GC Overhead limit\n# exceeded exceptions. If the oozie server is handling large number of\n# workflows/coordinator jobs, the memory setting
s may need to be revised\nexport CATALINA_OPTS=\"${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m \"",
+ "oozie_ambari_database": {"remove": "yes"},
+ "oozie_existing_mysql_database": {"remove": "yes"},
+ "oozie_existing_mysql_host": {"remove": "yes"},
+ "oozie_existing_oracle_database": {"remove": "yes"}
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.SchemaService.wf.ext.schemas": "shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.services": "\n org.apache.oozie.service.SchedulerService,\n org.apache.oozie.service.InstrumentationService,\n org.apache.oozie.service.MemoryLocksService,\n org.apache.oozie.service.UUIDService,\n org.apache.oozie.service.ELService,\n org.apache.oozie.service.AuthorizationService,\n org.apache.oozie.service.UserGroupInformationService,\n org.apache.oozie.service.HadoopAccessorService,\n org.apache.oozie.service.JobsConcurrencyService,\n org.apache.oozie.service.URIHandlerService,\n org.apache.oozie.service.DagXLogInfoService,\n org.apache.oozie.service.SchemaService,\n org.apache.oozie.service.LiteWorkflowAppService,\n org.apache.oozie.service.JPAService,\n org.apache.oozie.service.StoreService,\n org.apache.oozie.service.CoordinatorStoreService,\n org.apache.oozie.service.SLAStoreService,\n org.apache.oozie.service.DBLiteWorkflowStoreService,\n org.apache.oozie.service.Callbac
kService,\n org.apache.oozie.service.ShareLibService,\n org.apache.oozie.service.CallableQueueService,\n org.apache.oozie.service.ActionService,\n org.apache.oozie.service.ActionCheckerService,\n org.apache.oozie.service.RecoveryService,\n org.apache.oozie.service.PurgeService,\n org.apache.oozie.service.CoordinatorEngineService,\n org.apache.oozie.service.BundleEngineService,\n org.apache.oozie.service.DagEngineService,\n org.apache.oozie.service.CoordMaterializeTriggerService,\n org.apache.oozie.service.StatusTransitService,\n org.apache.oozie.service.PauseTransitService,\n org.apache.oozie.service.GroupsService,\n org.apache.oozie.service.ProxyUserService,\n org.apache.oozie.service.XLogStreamingService,\n org.apache.oozie.service.JvmPauseMonitorService"
+ },
+ "storm-env": {
+ "content": "\n#!/bin/bash\n\n# Set Storm specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\n\n# export STORM_CONF_DIR=\"\"\nexport STORM_HOME=/usr/hdp/current/storm-client"
+ },
+ "storm-site": {
+ "_storm.min.ruid": "null",
+ "_storm.thrift.nonsecure.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
+ "_storm.thrift.secure.transport": "backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin",
+ "drpc.childopts": "-Xmx768m _JAAS_PLACEHOLDER",
+ "java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib:/usr/hdp/current/storm-client/lib",
+ "logviewer.childopts": "-Xmx128m _JAAS_PLACEHOLDER",
+ "nimbus.childopts": "-Xmx1024m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "storm.thrift.transport": {"remove": "yes"},
+ "supervisor.childopts": "-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "ui.childopts": "-Xmx768m _JAAS_PLACEHOLDER",
+ "worker.childopts": "-Xmx768m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-client/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-client/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM"
+ },
+ "tez-site": {
+ "tez.am.container.idle.release-timeout-max.millis": "20000",
+ "tez.am.container.idle.release-timeout-min.millis": "10000",
+ "tez.am.container.reuse.non-local-fallback.enabled": "false",
+ "tez.am.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.am.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.am.max.app.attempts": "2",
+ "tez.am.maxtaskfailures.per.node": "10",
+ "tez.am.resource.memory.mb": "1364",
+ "tez.am.tez-ui.history-url.template": "__HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__",
+ "tez.cluster.additional.classpath.prefix": "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "tez.counters.max": "2000",
+ "tez.counters.max.groups": "1000",
+ "tez.generate.debug.artifacts": "false",
+ "tez.grouping.max-size": "1073741824",
+ "tez.grouping.min-size": "16777216",
+ "tez.grouping.split-waves": "1.7",
+ "tez.history.logging.service.class": "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService",
+ "tez.lib.uris": "/hdp/apps/${hdp.version}/tez/tez.tar.gz",
+ "tez.runtime.compress": "true",
+ "tez.runtime.compress.codec": "org.apache.hadoop.io.compress.SnappyCodec",
+ "tez.runtime.convert.user-payload.to.history-text": "false",
+ "tez.runtime.io.sort.mb": "272",
+ "tez.runtime.unordered.output.buffer.size-mb": "51",
+ "tez.session.client.timeout.secs": "-1",
+ "tez.shuffle-vertex-manager.max-src-fraction": "0.4",
+ "tez.shuffle-vertex-manager.min-src-fraction": "0.2",
+ "tez.task.am.heartbeat.counter.interval-ms.max": "4000",
+ "tez.task.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.task.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.task.max-events-per-heartbeat": "500",
+ "tez.task.resource.memory.mb": "682",
+ "tez.use.cluster.hadoop-libs": "false",
+ "tez.am.container.session.delay-allocation-millis": {"remove": "yes"},
+ "tez.am.grouping.max-size": {"remove": "yes"},
+ "tez.am.grouping.min-size": {"remove": "yes"},
+ "tez.am.grouping.split-waves": {"remove": "yes"},
+ "tez.am.java.opts": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.max-src-fraction": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.min-src-fraction": {"remove": "yes"},
+ "tez.lib.uris": "/hdp/apps/${hdp.version}/tez/tez.tar.gz",
+ "tez.runtime.intermediate-output.compress.codec": {"remove": "yes"},
+ "tez.use.cluster.hadoop-libs": "false",
+ "tez.yarn.ats.enabled": {"remove": "yes"},
+ "tez.runtime.intermediate-input.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-input.is-compressed": {"remove": "yes"},
+ "tez.runtime.intermediate-output.should-compress": {"remove": "yes"},
+ "tez.am.env": {"remove": "yes"}
+ },
+ "webhcat-log4j": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Define some default values that can be overridden by system properties\nwebhcat.root.logger = INFO, standard\nwebhcat.log.dir = .\nwebhcat.log.file = webhcat.log\n\nlog
4j.rootLogger = ${webhcat.root.logger}\n\n# Logging Threshold\nlog4j.threshhold = DEBUG\n\nlog4j.appender.standard = org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern = .yyyy-MM-dd\n\nlog4j.appender.DRFA.layout = org.apache.log4j.PatternLayout\n\nlog4j.appender.standard.layout = org.apache.log4j.PatternLayout\nlog4j.appender.standard.layout.conversionPattern = %-5p | %d{DATE} | %c | %m%n\n\n# Class logging settings\nlog4j.logger.com.sun.jersey = DEBUG\nlog4j.logger.com.sun.jersey.spi.container.servlet.WebComponent = ERROR\nlog4j.logger.org.apache.hadoop = INFO\nlog4j.logger.org.apache.hadoop.conf = WARN\nlog4j.logger.org.apache.zookeeper = WARN\nlog4j.logger.org.eclipse.jetty = INFO"
+ },
+ "webhcat-site": {
+ "templeton.hadoop": "/usr/hdp/current/hadoop-client/bin/hadoop",
+ "templeton.hcat": "/usr/hdp/current/hive-client/bin/hcat",
+ "templeton.hive.archive": "hdfs:///hdp/apps/${hdp.version}/hive/hive.tar.gz",
+ "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+ "templeton.libjars": "/usr/hdp/current/zookeeper-client/zookeeper.jar",
+ "templeton.pig.archive": "hdfs:///hdp/apps/${hdp.version}/pig/pig.tar.gz",
+ "templeton.sqoop.archive": "hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop.tar.gz",
+ "templeton.sqoop.home": "sqoop.tar.gz/sqoop",
+ "templeton.sqoop.path": "sqoop.tar.gz/sqoop/bin/sqoop",
+ "templeton.streaming.jar": "hdfs:///hdp/apps/${hdp.version}/mapreduce/hadoop-streaming.jar"
+ },
+ "yarn-site": {
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "yarn.client.nodemanager-connect.max-wait-ms": "60000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-per-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "{{yarn_log_dir_prefix}}/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "true",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "true",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000"
+ },
+ "hiveserver2-site": {
+ "hive.security.authenticator.manager": "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
+ },
+ "ranger-hbase-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hbase",
+ "REPOSITORY_CONFIG_USERNAME": "hbase",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "policy_user": "ambari-qa",
+ "ranger-hbase-plugin-enabled": "No"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hadoop",
+ "REPOSITORY_CONFIG_USERNAME": "hadoop",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "hadoop.rpc.protection": "-",
+ "policy_user": "ambari-qa",
+ "ranger-hdfs-plugin-enabled": "No"
+ },
+ "ranger-hive-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hive",
+ "REPOSITORY_CONFIG_USERNAME": "hive",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver",
+ "policy_user": "ambari-qa",
+ "ranger-hive-plugin-enabled": "No"
+ },
+ "ranger-storm-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "stormtestuser",
+ "REPOSITORY_CONFIG_USERNAME": "stormtestuser@EXAMPLE.COM",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "policy_user": "storm",
+ "ranger-storm-plugin-enabled": "No"
+ }
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.json
new file mode 100644
index 0000000..207bc58
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.json
@@ -0,0 +1,292 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.1",
+ "target-version": "2.2",
+ "options": {
+ "config-types": {
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "tez-site":{
+ "merged-copy": "yes"
+ },
+ "falcon-startup.properties": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "falcon-startup.properties": {
+ "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService",
+ "*.dfs.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+ "*.falcon.enableTLS": "false",
+ "*.falcon.http.authentication.cookie.domain": "EXAMPLE.COM",
+ "*.falcon.http.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "*.falcon.http.authentication.kerberos.principal": "HTTP/_HOST@EXAMPLE.COM",
+ "*.falcon.security.authorization.admin.groups": "falcon",
+ "*.falcon.security.authorization.admin.users": "falcon,ambari-qa",
+ "*.falcon.security.authorization.enabled": "false",
+ "*.falcon.security.authorization.provider": "org.apache.falcon.security.DefaultAuthorizationProvider",
+ "*.falcon.security.authorization.superusergroup": "falcon",
+ "*.falcon.service.authentication.kerberos.keytab": "/etc/security/keytabs/falcon.service.keytab",
+ "*.falcon.service.authentication.kerberos.principal": "falcon/_HOST@EXAMPLE.COM",
+ "*.journal.impl": "org.apache.falcon.transaction.SharedFileSystemJournal",
+ "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
+ "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph, org.apache.falcon.entity.ColoClusterRelation, org.apache.falcon.group.FeedGroupMap"
+ },
+ "tez-site":{
+ "tez.am.container.idle.release-timeout-max.millis": "20000",
+ "tez.am.container.idle.release-timeout-min.millis": "10000",
+ "tez.am.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.am.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.am.max.app.attempts": "2",
+ "tez.am.maxtaskfailures.per.node": "10",
+ "tez.cluster.additional.classpath.prefix": "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "tez.counters.max": "2000",
+ "tez.counters.max.groups": "1000",
+ "tez.generate.debug.artifacts": "false",
+ "tez.grouping.max-size": "1073741824",
+ "tez.grouping.min-size": "16777216",
+ "tez.grouping.split-waves": "1.7",
+ "tez.history.logging.service.class": "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService",
+ "tez.runtime.compress": "true",
+ "tez.runtime.compress.codec": "org.apache.hadoop.io.compress.SnappyCodec",
+ "tez.runtime.io.sort.mb": "272",
+ "tez.runtime.unordered.output.buffer.size-mb": "51",
+ "tez.shuffle-vertex-manager.max-src-fraction": "0.4",
+ "tez.shuffle-vertex-manager.min-src-fraction": "0.2",
+ "tez.task.am.heartbeat.counter.interval-ms.max": "4000",
+ "tez.task.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.task.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.task.max-events-per-heartbeat": "500",
+ "tez.task.resource.memory.mb": "682",
+ "tez.am.container.reuse.non-local-fallback.enabled": "false",
+ "tez.am.resource.memory.mb": "1364",
+ "tez.lib.uris": "/hdp/apps/${hdp.version}/tez/tez.tar.gz",
+ "tez.session.client.timeout.secs": "-1",
+ "tez.am.container.session.delay-allocation-millis": {"remove": "yes"},
+ "tez.am.env": {"remove": "yes"},
+ "tez.am.grouping.max-size": {"remove": "yes"},
+ "tez.am.grouping.min-size": {"remove": "yes"},
+ "tez.am.grouping.split-waves": {"remove": "yes"},
+ "tez.am.java.opt": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.max-src-fraction": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.min-src-fraction": {"remove": "yes"},
+ "tez.runtime.intermediate-input.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-input.is-compressed": {"remove": "yes"},
+ "tez.runtime.intermediate-output.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-output.should-compress": {"remove": "yes"},
+ "tez.yarn.ats.enabled": {"remove": "yes"}
+ },
+ "webhcat-site": {
+ "templeton.storage.class": "org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage",
+ "templeton.port": "50111"
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.service.HadoopAccessorService.kerberos.enabled": "false",
+ "oozie.service.SchemaService.wf.ext.schemas": "shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd",
+ "oozie.services.ext": "org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService"
+ },
+ "hive-site": {
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.transport.mode": "binary",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.optimize.reducededuplication.min.reducer": "4",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.support.dynamic.service.discovery": "true",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "fs.file.impl.disable.cache": "true",
+ "fs.hdfs.impl.disable.cache": "true"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {"remove": "yes"}
+ },
+ "mapred-site": {
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64"
+ },
+ "core-site": {
+ "hadoop.http.authentication.simple.anonymous.allowed": "true"
+ },
+ "hdfs-site": {
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.datanode.max.transfer.threads": "4096"
+ },
+ "yarn-site": {
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.client.nodemanager-connect.max-wait-ms": "900000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strictresource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-er-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "/var/log/hadoop-yarn/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "false",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "false",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-address": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000"
+ }
+ }
+ }
+ ]
+}
[06/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json
new file mode 100755
index 0000000..a2c1ae1
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.4.json
@@ -0,0 +1,453 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.0",
+ "target-version": "2.2.4",
+ "options": {
+ "config-types": {
+ "capacity-scheduler": {
+ "merged-copy": "yes"
+ },
+ "cluster-env": {
+ "merged-copy": "yes"
+ },
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "flume-env": {
+ "merged-copy": "yes"
+ },
+ "hadoop-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-log4j": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "hive-env": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-env": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "oozie-env": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-log4j": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ },
+ "pig-properties": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "pig-properties": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.\n# see bin/pig -help\n\n# brief logging (no ti
mestamps)\nbrief=false\n\n# debug level, INFO is default\ndebug=INFO\n\n# verbose print all log messages to screen (default to print only INFO and above to screen)\nverbose=false\n\n# exectype local|mapreduce, mapreduce is default\nexectype=mapreduce\n\n# Enable insertion of information about script into hadoop job conf \npig.script.info.enabled=true\n\n# Do not spill temp files smaller than this size (bytes)\npig.spill.size.threshold=5000000\n\n# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)\n# This should help reduce the number of files being spilled.\npig.spill.gc.activation.size=40000000\n\n# the following two parameters are to help estimate the reducer number\npig.exec.reducers.bytes.per.reducer=1000000000\npig.exec.reducers.max=999\n\n# Temporary location to store the intermediate data.\npig.temp.dir=/tmp/\n\n# Threshold for merging FRJoin fragment files\npig.files.concatenation.threshold=100\npig.optimistic.files.concatenation=fals
e;\n\npig.disable.counter=false\n\n# Avoid pig failures when multiple jobs write to the same location\npig.location.check.strict=false\n\nhcat.bin=/usr/bin/hcat"
+ },
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+ "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.default-node-label-expression": " "
+ },
+ "cluster-env": {
+ "smokeuser_principal_name": {
+ "remove": "yes"
+ }
+ },
+ "core-site": {
+ "hadoop.http.authentication.simple.anonymous.allowed": "true",
+ "hadoop.proxyuser.falcon.groups": "users",
+ "hadoop.proxyuser.falcon.hosts": "*",
+ "hadoop.security.auth_to_local": "\n DEFAULT"
+ },
+ "flume-env":{
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced\n# during Flume startup.\n\n# Enviroment variables can be set here.\n\nexport JAVA_HOM
E={{java_home}}\n\n# Give Flume more memory and pre-allocate, enable remote monitoring via JMX\n# export JAVA_OPTS=\"-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote\"\n\n# Note that the Flume conf directory is always included in the classpath.\n# Add flume sink to classpath\nif [ -e \"/usr/lib/flume/lib/ambari-metrics-flume-sink.jar\" ]; then\n export FLUME_CLASSPATH=$FLUME_CLASSPATH:/usr/lib/flume/lib/ambari-metrics-flume-sink.jar\nfi\n\nexport HIVE_HOME={{flume_hive_home}}\nexport HCAT_HOME={{flume_hcat_home}}"
+ },
+ "hadoop-env": {
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options
appended to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGC
DateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The f
ollowing applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# ex
port HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /us
r/share/java/*mysql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n if [ -d \"/etc/tez/conf/\" ]; then\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n fi\nfi\n\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION
$HADOOP_OPTS\"",
+ "namenode_opt_maxnewsize": "256m",
+ "namenode_opt_newsize": "256m"
+ },
+ "hbase-env": {
+ "content": "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to
enable JMX exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG
_DIR={{log_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport HBASE_RE
GIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {
+ "remove": "yes"
+ },
+ "hbase.hregion.majorcompaction.jitter": "0.50"
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop
.root.logger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.
appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.secur
ity.logger}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesyste
m.audit=false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoo
p.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metri
cs.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN"
+ },
+ "hdfs-site": {
+ "dfs.datanode.max.transfer.threads": "16384",
+ "dfs.namenode.handler.count": "100",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600"
+ },
+ "hive-env": {
+ "content": "\n if [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra libraries required f
or hive compilation/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n if [ -f \"${HIVE_AUX_JARS_PATH}\" ]; then \n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\n elif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\n fi\nelif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\nfi \n\nexport METASTORE_PORT={{hive_metastore_port}}"
+ },
+ "hive-site": {
+ "fs.file.impl.disable.cache": {
+ "remove": "yes"
+ },
+ "fs.hdfs.impl.disable.cache": {
+ "remove": "yes"
+ },
+ "hive.heapsize": {
+ "remove": "yes"
+ },
+ "hive.optimize.mapjoin.mapreduce": {
+ "remove": "yes"
+ },
+ "hive.server2.enable.impersonation": {
+ "remove": "yes"
+ },
+ "datanucleus.cache.level2.type": "none",
+ "hive.auto.convert.join.noconditionaltask.size": "238026752",
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.auto.convert.sortmerge.join.noconditionaltask": {
+ "remove": "yes"
+ },
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.compactor.abortedtxn.threshold": "1000",
+ "hive.compactor.check.interval": "300L",
+ "hive.compactor.delta.num.threshold": "10",
+ "hive.compactor.delta.pct.threshold": "0.1f",
+ "hive.compactor.initiator.on": "false",
+ "hive.compactor.worker.threads": "0",
+ "hive.compactor.worker.timeout": "86400L",
+ "hive.compute.query.using.stats": "true",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.enforce.sortmergebucketmapjoin": "true",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.failure.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.post.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.pre.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.execution.engine": "tez",
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.limit.optimize.enable": "true",
+ "hive.limit.pushdown.memory.usage": "0.04",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.kerberos.keytab.file": "/etc/security/keytabs/hive.service.keytab",
+ "hive.metastore.kerberos.principal": "hive/_HOST@EXAMPLE.COM",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.reducededuplication.min.reducer": "4",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.orc.splits.include.file.footer": "false",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.authentication.spnego.keytab": "HTTP/_HOST@EXAMPLE.COM",
+ "hive.server2.authentication.spnego.principal": "/etc/security/keytabs/spnego.service.keytab",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.support.dynamic.service.discovery": "true",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.tez.default.queues": "default",
+ "hive.server2.tez.initialize.default.sessions": "false",
+ "hive.server2.tez.sessions.per.default.queue": "1",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.autogather": "true",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.container.size": "682",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.input.format": "org.apache.hadoop.hive.ql.io.HiveInputFormat",
+ "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager",
+ "hive.txn.max.open.batch": "1000",
+ "hive.txn.timeout": "300",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.enabled": "true",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "hive.vectorized.groupby.flush.percent": "0.1",
+ "hive.vectorized.groupby.maxentries": "100000",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace"
+ },
+ "hiveserver2-site": {
+ "hive.security.authenticator.manager": "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
+ },
+ "mapred-env": {
+ "content": "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n#export HADOOP_JOB_HISTORYSERVER_OPTS=\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\""
+ },
+ "mapred-site": {
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}"
+ },
+ "oozie-env": {
+ "content": "\n#!/bin/bash\n\nif [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\nfi\n\n#Set JAVA HOME\nexport JAVA_HOME={{java_home}}\n\nexport JRE_HOME=${JAVA_HOME}\n\n# Set Oozie specific environment variables here.\n\n# Settings for the Embedded Tomcat that runs Oozie\n# Java System properties for Oozie should be specified in this variable\n#\n# export CATALINA_OPTS=\n\n# Oozie configuration file to load from Oozie configuration directory\n#\n# export OOZIE_CONFIG_FILE=oozie-site.xml\n\n# Oozie logs directory\n#\nexport OOZIE_LOG={{oozie_log_dir}}\n\n# Oozie pid directory\n#\nexport CATALINA_PID={{pid_file}}\n\n#Location of the data for oozie\nexport OOZIE_DATA={{oozie_data_dir}}\n\n# Oozie Log4J configuration file to load from Oozie confi
guration directory\n#\n# export OOZIE_LOG4J_FILE=oozie-log4j.properties\n\n# Reload interval of the Log4J configuration file, in seconds\n#\n# export OOZIE_LOG4J_RELOAD=10\n\n# The port Oozie server runs\n#\nexport OOZIE_HTTP_PORT={{oozie_server_port}}\n\n# The admin port Oozie server runs\n#\nexport OOZIE_ADMIN_PORT={{oozie_server_admin_port}}\n\n# The host name Oozie server runs on\n#\n# export OOZIE_HTTP_HOSTNAME=`hostname -f`\n\n# The base URL for callback URLs to Oozie\n#\n# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\nexport JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64\n\n# At least 1 minute of retry time to account for server downtime during\n# upgrade/downgrade\nexport OOZIE_CLIENT_OPTS=\"${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 \"\n\n# This is needed so that Oozie does not run into OOM or GC Overhead limit\n# exceeded exceptions. If the oozie server is handling large number of\n# workflows/coordinator jobs, th
e memory settings may need to be revised\nexport CATALINA_OPTS=\"${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m \"",
+ "oozie_ambari_database":{
+ "remove": "yes"
+ }
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.ELService.ext.functions.coord-action-create": "\n now=org.apache.oozie.extensions.OozieELExtensions#ph2_now,\n today=org.apache.oozie.extensions.OozieELExtensions#ph2_today,\n yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday,\n currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth,\n lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth,\n currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear,\n lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear,\n latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,\n future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,\n formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,\n user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-action-create-inst": "\n now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst,\n today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst,\n yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst,\n currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst,\n lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst,\n currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst,\n lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst,\n latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,\n future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,\n formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,\n user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-action-start": "\n now=org.apache.oozie.extensions.OozieELExtensions#ph2_now,\n today=org.apache.oozie.extensions.OozieELExtensions#ph2_today,\n yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday,\n currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth,\n lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth,\n currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear,\n lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear,\n latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest,\n future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future,\n dataIn=org.apache.oozie.extensions.OozieELExtensions#ph3_dataIn,\n instanceTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime,\n dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset,\n
formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime,\n user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-job-submit-data": "\n now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo,\n today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo,\n yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo,\n currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo,\n lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo,\n currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo,\n lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo,\n dataIn=org.apache.oozie.extensions.OozieELExtensions#ph1_dataIn_echo,\n instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap,\n formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,\n dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,\n
user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-job-submit-instances": "\n now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo,\n today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo,\n yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo,\n currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo,\n lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo,\n currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo,\n lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo,\n formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,\n latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,\n future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo",
+ "oozie.service.ELService.ext.functions.coord-sla-create": "\n instanceTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime,\n user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-sla-submit": "\n instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed,\n user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.HadoopAccessorService.supported.filesystems": "*",
+ "oozie.service.SchemaService.wf.ext.schemas": "shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.services": "\n org.apache.oozie.service.SchedulerService,\n org.apache.oozie.service.InstrumentationService,\n org.apache.oozie.service.MemoryLocksService,\n org.apache.oozie.service.UUIDService,\n org.apache.oozie.service.ELService,\n org.apache.oozie.service.AuthorizationService,\n org.apache.oozie.service.UserGroupInformationService,\n org.apache.oozie.service.HadoopAccessorService,\n org.apache.oozie.service.JobsConcurrencyService,\n org.apache.oozie.service.URIHandlerService,\n org.apache.oozie.service.DagXLogInfoService,\n org.apache.oozie.service.SchemaService,\n org.apache.oozie.service.LiteWorkflowAppService,\n org.apache.oozie.service.JPAService,\n org.apache.oozie.service.StoreService,\n org.apache.oozie.service.CoordinatorStoreService,\n org.apache.oozie.service.SLAStoreService,\n org.apache.oozie.service.DBLiteWorkflowStoreService,\n org.apache.oozie
.service.CallbackService,\n org.apache.oozie.service.ShareLibService,\n org.apache.oozie.service.CallableQueueService,\n org.apache.oozie.service.ActionService,\n org.apache.oozie.service.ActionCheckerService,\n org.apache.oozie.service.RecoveryService,\n org.apache.oozie.service.PurgeService,\n org.apache.oozie.service.CoordinatorEngineService,\n org.apache.oozie.service.BundleEngineService,\n org.apache.oozie.service.DagEngineService,\n org.apache.oozie.service.CoordMaterializeTriggerService,\n org.apache.oozie.service.StatusTransitService,\n org.apache.oozie.service.PauseTransitService,\n org.apache.oozie.service.GroupsService,\n org.apache.oozie.service.ProxyUserService,\n org.apache.oozie.service.XLogStreamingService,\n org.apache.oozie.service.JvmPauseMonitorService",
+ "oozie.services.ext": "org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService"
+ },
+ "ranger-hbase-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hbase",
+ "REPOSITORY_CONFIG_USERNAME": "hbase",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "policy_user": "ambari-qa",
+ "ranger-hbase-plugin-enabled": "No"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hadoop",
+ "REPOSITORY_CONFIG_USERNAME": "hadoop",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "hadoop.rpc.protection": "-",
+ "policy_user": "ambari-qa",
+ "ranger-hdfs-plugin-enabled": "No"
+ },
+ "ranger-hive-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hive",
+ "REPOSITORY_CONFIG_USERNAME": "hive",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver",
+ "policy_user": "ambari-qa",
+ "ranger-hive-plugin-enabled": "No"
+ },
+ "webhcat-log4j": {
+ "content": "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Define some default values that can be overridden by system properties\nwebhcat.root.logger = INFO, standard\nwebhcat.log.dir = .\nwebhcat.log.file = we
bhcat.log\n\nlog4j.rootLogger = ${webhcat.root.logger}\n\n# Logging Threshold\nlog4j.threshhold = DEBUG\n\nlog4j.appender.standard = org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern = .yyyy-MM-dd\n\nlog4j.appender.DRFA.layout = org.apache.log4j.PatternLayout\n\nlog4j.appender.standard.layout = org.apache.log4j.PatternLayout\nlog4j.appender.standard.layout.conversionPattern = %-5p | %d{DATE} | %c | %m%n\n\n# Class logging settings\nlog4j.logger.com.sun.jersey = DEBUG\nlog4j.logger.com.sun.jersey.spi.container.servlet.WebComponent = ERROR\nlog4j.logger.org.apache.hadoop = INFO\nlog4j.logger.org.apache.hadoop.conf = WARN\nlog4j.logger.org.apache.zookeeper = WARN\nlog4j.logger.org.eclipse.jetty = INFO"
+ },
+ "webhcat-site": {
+ "templeton.hadoop": "/usr/hdp/current/hadoop-client/bin/hadoop",
+ "templeton.hcat": "/usr/hdp/current/hive-client/bin/hcat",
+ "templeton.hive.archive": "hdfs:///hdp/apps/${hdp.version}/hive/hive.tar.gz",
+ "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+ "templeton.libjars": "/usr/hdp/current/zookeeper-client/zookeeper.jar",
+ "templeton.pig.archive": "hdfs:///hdp/apps/${hdp.version}/pig/pig.tar.gz",
+ "templeton.sqoop.archive": "hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop.tar.gz",
+ "templeton.sqoop.home": "sqoop.tar.gz/sqoop",
+ "templeton.sqoop.path": "sqoop.tar.gz/sqoop/bin/sqoop",
+ "templeton.streaming.jar": "hdfs:///hdp/apps/${hdp.version}/mapreduce/hadoop-streaming.jar"
+ },
+ "yarn-site": {
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "yarn.client.nodemanager-connect.max-wait-ms": "60000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-per-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "{{yarn_log_dir_prefix}}/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "true",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "true",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.generic-application-history.store-class": "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.path": "/hadoop/yarn/timeline",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms": "300000",
+ "yarn.timeline-service.store-class": "org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore",
+ "yarn.timeline-service.ttl-enable": "true",
+ "yarn.timeline-service.ttl-ms": "2678400000"
+ }
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.json
new file mode 100644
index 0000000..798e276
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.json
@@ -0,0 +1,275 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.0",
+ "target-version": "2.2",
+ "options": {
+ "config-types": {
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "webhcat-site": {
+ "templeton.storage.class": "org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage",
+ "templeton.port": "50111"
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.service.ELService.ext.functions.coord-action-create": "now=org.apache.oozie.extensions.OozieELExtensions#ph2_now,today=org.apache.oozie.extensions.OozieELExtensions#ph2_today,yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday,currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth,lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth,currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear,lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear,latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-action-create-inst": "now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst,today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst,yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst,currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst,lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst,currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst,lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst,latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-action-start": "now=org.apache.oozie.extensions.OozieELExtensions#ph2_now,today=org.apache.oozie.extensions.OozieELExtensions#ph2_today,yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday,currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth,lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth,currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear,lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear,latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest,future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future,dataIn=org.apache.oozie.extensions.OozieELExtensions#ph3_dataIn,instanceTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime,dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset,formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime,user=org.apache.oozie.coord.CoordELFu
nctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-job-submit-data": "now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo,today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo,yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo,currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo,lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo,currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo,lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo,dataIn=org.apache.oozie.extensions.OozieELExtensions#ph1_dataIn_echo,instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap,formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-job-submit-instances": "now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo,today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo,yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo,currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo,lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo,currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo,lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo,formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo",
+ "oozie.service.ELService.ext.functions.coord-sla-create": "instanceTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime,user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.ELService.ext.functions.coord-sla-submit": "instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed,user=org.apache.oozie.coord.CoordELFunctions#coord_user",
+ "oozie.service.HadoopAccessorService.kerberos.enabled": "false",
+ "oozie.service.HadoopAccessorService.supported.filesystems": "*",
+ "oozie.service.SchemaService.wf.ext.schemas": "shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd",
+ "oozie.services.ext": "org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService"
+ },
+ "mapred-site": {
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.map.java.opts": "-Xmx546m",
+ "mapreduce.map.memory.mb": "682",
+ "mapreduce.reduce.java.opts": "-Xmx546m",
+ "mapreduce.task.io.sort.mb": "273",
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.resource.mb": "682",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64"
+ },
+ "yarn-site": {
+ "yarn.timeline-service.leveldb-timeline-store.path": "/var/log/hadoop-yarn/timeline",
+ "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms": "300000",
+ "yarn.timeline-service.store-class": "org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore",
+ "yarn.timeline-service.ttl-enable": "true",
+ "yarn.timeline-service.ttl-ms": "2678400000",
+ "yarn.timeline-service.generic-application-history.store-class": "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore",
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.client.nodemanager-connect.max-wait-ms": "900000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-per-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "/var/log/hadoop-yarn/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "false",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "false",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-address": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.enabled": "true",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000"
+ },
+ "hive-site": {
+ "hive.execution.engine": "mr",
+ "hive.exec.failure.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.post.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.pre.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "datanucleus.cache.level2.type": "none",
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.compactor.abortedtxn.threshold": "1000",
+ "hive.compactor.check.interval": "300L",
+ "hive.compactor.delta.num.threshold": "10",
+ "hive.compactor.delta.pct.threshold": "0.1f",
+ "hive.compactor.initiator.on": "false",
+ "hive.compactor.worker.threads": "0",
+ "hive.compactor.worker.timeout": "86400L",
+ "hive.compute.query.using.stats": "true",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.enforce.sortmergebucketmapjoin": "true",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.limit.optimize.enable": "true",
+ "hive.limit.pushdown.memory.usage": "0.04",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.kerberos.keytab.file": "/etc/security/keytabs/hive.service.keytab",
+ "hive.metastore.kerberos.principal": "hive/_HOST@EXAMPLE.COM",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.orc.splits.include.file.footer": "false",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.authentication.spnego.keytab": "HTTP/_HOST@EXAMPLE.COM",
+ "hive.server2.authentication.spnego.principal": "/etc/security/keytabs/spnego.service.keytab",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.tez.default.queues": "default",
+ "hive.server2.tez.sessions.per.default.queue": "1",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.transport.mode": "binary",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.autogather": "true",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.input.format": "org.apache.hadoop.hive.ql.io.HiveInputFormat",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager",
+ "hive.txn.max.open.batch": "1000",
+ "hive.txn.timeout": "300",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "hive.vectorized.groupby.flush.percent": "0.1",
+ "hive.vectorized.groupby.maxentries": "100000",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "hive.auto.convert.join.noconditionaltask.size": "238026752",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.optimize.reducededuplication.min.reducer": "4",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.support.dynamic.service.discovery": "true",
+ "hive.tez.container.size": "682",
+ "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
+ "fs.file.impl.disable.cache": "true",
+ "fs.hdfs.impl.disable.cache": "true"
+ },
+ "core-site": {
+ "hadoop.proxyuser.falcon.groups": "users",
+ "hadoop.proxyuser.falcon.hosts": "*"
+ },
+ "hdfs-site": {
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.datanode.max.transfer.threads": "4096"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {"remove": "yes"}
+ }
+ }
+ }
+ ]
+}
[53/55] [abbrv] ambari git commit: AMBARI-18193. Zeppelin logsearch
configuration typo (Miklos Gergely via oleewere)
Posted by ol...@apache.org.
AMBARI-18193. Zeppelin logsearch configuration typo (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7ebdca6b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7ebdca6b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7ebdca6b
Branch: refs/heads/branch-dev-logsearch
Commit: 7ebdca6b240e7d678332e2f0ccacfd80585aaffd
Parents: 0567b17
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Mon Aug 22 10:59:02 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:03 2016 +0200
----------------------------------------------------------------------
.../common-services/LOGSEARCH/0.5.0/package/scripts/params.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7ebdca6b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index cb7b79f..5c3954f 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -205,7 +205,7 @@ ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ra
storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm')
yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')
mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')
-zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
+zeppelin_log_dir = default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')
spark_log_dir = default('/configurations/spark-env/spark_log_dir', '/var/log/spark')
livy_log_dir = default('/configurations/livy-env/livy_log_dir', '/var/log/livy')
[52/55] [abbrv] ambari git commit: AMBARI-18214. Restify Log Search
endpoints (oleewere)
Posted by ol...@apache.org.
AMBARI-18214. Restify Log Search endpoints (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0567b179
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0567b179
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0567b179
Branch: refs/heads/branch-dev-logsearch
Commit: 0567b179fe1a06f13759e8e645c8d499a0ad43cf
Parents: f1f5ea6
Author: oleewere <ol...@gmail.com>
Authored: Fri Aug 19 17:31:55 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:03 2016 +0200
----------------------------------------------------------------------
.../ambari/logsearch/dao/SolrDaoBase.java | 2 +-
.../ambari/logsearch/doc/DocConstants.java | 1 +
.../ambari/logsearch/manager/AuditMgr.java | 2 +-
.../ambari/logsearch/manager/LogsMgr.java | 119 ++--
.../ambari/logsearch/rest/AuditLogsREST.java | 283 +++++++++
.../apache/ambari/logsearch/rest/AuditREST.java | 312 ----------
.../ambari/logsearch/rest/DashboardREST.java | 575 ------------------
.../ambari/logsearch/rest/LogFileREST.java | 5 +-
.../ambari/logsearch/rest/PublicREST.java | 2 +-
.../ambari/logsearch/rest/ServiceLogsREST.java | 578 +++++++++++++++++++
.../ambari/logsearch/rest/UserConfigREST.java | 17 +-
.../src/main/webapp/login.html | 28 +-
.../collection_bases/VAuditLogListBase.js | 16 +-
.../collection_bases/VEventHistoryListBase.js | 14 +-
.../scripts/collection_bases/VGroupListBase.js | 16 +-
.../collection_bases/VLogLevelListBase.js | 2 +-
.../scripts/collection_bases/VLogListBase.js | 10 +-
.../collection_bases/VNameValueListBase.js | 14 +-
.../scripts/collection_bases/VNodeListBase.js | 4 +-
.../webapp/scripts/model_bases/VAuditLogBase.js | 2 +-
.../scripts/model_bases/VCommonModelBase.js | 2 +-
.../scripts/model_bases/VEventHistoryBase.js | 2 +-
.../scripts/model_bases/VGraphInfoBase.js | 2 +-
.../webapp/scripts/model_bases/VLogLevelBase.js | 2 +-
.../scripts/model_bases/VUserFilterBase.js | 2 +-
.../src/main/webapp/scripts/utils/ViewUtils.js | 2 +-
.../scripts/views/audit/AuditAggregatedView.js | 6 +-
.../scripts/views/audit/AuditTabLayoutView.js | 22 +-
.../dashboard/BubbleGraphTableLayoutView.js | 30 +-
.../views/dashboard/ComponentListView.js | 4 +-
.../scripts/views/dashboard/ComponentsView.js | 2 +-
.../scripts/views/dashboard/DashboardView.js | 12 +-
.../views/dashboard/GridTableLayoutView.js | 2 +-
.../webapp/scripts/views/dashboard/HostsView.js | 2 +-
.../scripts/views/dashboard/LogLevelBoxView.js | 2 +-
.../scripts/views/dashboard/MainLayoutView.js | 44 +-
.../dialog/GlobalExclusionCompositeView.js | 2 +-
.../views/filter/CreateLogfeederFilterView.js | 20 +-
.../scripts/views/graphs/GraphLayoutView.js | 12 +-
.../scripts/views/graphs/GridGraphLayoutView.js | 2 +-
.../webapp/scripts/views/tabs/ComparisonView.js | 2 +-
.../views/tabs/HierarchyTabLayoutView.js | 36 +-
.../webapp/scripts/views/tabs/LogFileView.js | 12 +-
.../main/webapp/scripts/views/tabs/TreeView.js | 2 +-
.../troubleshoot/TroubleShootLayoutView.js | 6 +-
.../src/main/webapp/templates/graphs/backup.js | 12 +-
.../logging/LoggingRequestHelperImpl.java | 4 +-
47 files changed, 1107 insertions(+), 1143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 672507a..91c4a26 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -547,7 +547,7 @@ public abstract class SolrDaoBase {
QueryResponse queryResponse = solrClient.query(solrQuery,
METHOD.POST);
- if (event != null && !"/getLiveLogsCount".equalsIgnoreCase(event)) {
+ if (event != null && !"/audit/logs/live/count".equalsIgnoreCase(event)) {
logPerformance.info("\n Username :- "
+ LogsearchContextUtil.getCurrentUsername()
+ " Event :- " + event + " SolrQuery :- " + solrQuery
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index c1572b7..c14fc08 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -105,6 +105,7 @@ public class DocConstants {
public static final String HOST_D = "";
public static final String COMPONENT_D = "";
public static final String LOG_TYPE_D = "";
+ public static final String TAIL_SIZE_D = "";
}
public class LogFileOperationDescriptions {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
index d4f2986..58c3a4d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
@@ -258,7 +258,7 @@ public class AuditMgr extends MgrBase {
public String getLiveLogCounts() {
VNameValueList nameValueList = new VNameValueList();
SolrQuery solrQuery = new SolrQuery();
- solrQuery.setParam("event", "/getLiveLogsCount");
+ solrQuery.setParam("event", "/audit/logs/live/count");
try {
String startDate = dateUtil
.convertGivenDateFormatToSolrDateFormat(ManageStartEndTime.startDate);
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
index b03a643..451fac1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -88,7 +88,7 @@ public class LogsMgr extends MgrBase {
public static List<String> cancelByDate = new CopyOnWriteArrayList<String>();
public static Map<String, String> mapUniqueId = new ConcurrentHashMap<String, String>();
-
+
public static enum CONDITION {
OR, AND
}
@@ -140,7 +140,7 @@ public class LogsMgr extends MgrBase {
SolrQuery solrQuery = queryGenerator
.commonServiceFilterQuery(searchCriteria);
- solrQuery.setParam("event", "/solr/logs_search");
+ solrQuery.setParam("event", "/service/logs");
VSolrLogList collection = getLogAsPaginationProvided(solrQuery,
serviceLogsSolrDao);
@@ -151,7 +151,7 @@ public class LogsMgr extends MgrBase {
public String getHosts(SearchCriteria searchCriteria) {
return getFields(searchCriteria, LogSearchConstants.SOLR_HOST);
}
-
+
public String getFields(SearchCriteria searchCriteria,String field){
SolrQuery solrQuery = new SolrQuery();
@@ -185,7 +185,7 @@ public class LogsMgr extends MgrBase {
solrDoc.put(field, temp);
docList.add(solrDoc);
}
-
+
collection.setGroupDocuments(docList);
if(!docList.isEmpty()){
collection.setStartIndex((int) docList.getStart());
@@ -197,7 +197,7 @@ public class LogsMgr extends MgrBase {
throw restErrorUtil.createRESTException(MessageEnums.SOLR_ERROR
.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
}
-
+
}
public String getComponents(SearchCriteria searchCriteria) {
@@ -274,11 +274,11 @@ public class LogsMgr extends MgrBase {
return collection;
}
List<Count> fieldList = facetFields.getValues();
-
+
if(fieldList == null){
return collection;
}
-
+
for (Count cnt : fieldList) {
if (cnt != null) {
VCount vCount = new VCount();
@@ -293,11 +293,11 @@ public class LogsMgr extends MgrBase {
throw restErrorUtil.createRESTException(MessageEnums.SOLR_ERROR
.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
}
-
+
collection.setCounts(vCounts);
return collection;
}
-
+
public VCountList getLogLevelCount(SearchCriteria searchCriteria) {
return getFieldCount(searchCriteria, LogSearchConstants.SOLR_LEVEL);
}
@@ -331,7 +331,7 @@ public class LogsMgr extends MgrBase {
if(!stringUtil.isEmpty(firstPriority)){
hostNode.setType(firstPriority);
}
-
+
hostNode.setParent(true);
hostNode.setRoot(true);
PivotField hostPivot = null;
@@ -467,7 +467,7 @@ public class LogsMgr extends MgrBase {
public String getHostListByComponent(SearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
- solrQuery.setParam("event", "/getHostListByComponent");
+ solrQuery.setParam("event", "/service/hosts/component");
VNodeList list = new VNodeList();
if (searchCriteria.getSortBy() == null) {
@@ -484,10 +484,10 @@ public class LogsMgr extends MgrBase {
} else {
return convertObjToString(list);
}
-
+
String firstHirarchy = "type,host,level";
String secondHirarchy = "type,level";
-
+
try {
queryGenerator.setFacetPivot(solrQuery, 1, firstHirarchy,
secondHirarchy);
@@ -528,7 +528,7 @@ public class LogsMgr extends MgrBase {
public VNameValueList getLogsLevelCount(SearchCriteria sc) {
VNameValueList nameValueList = new VNameValueList();
SolrQuery query = queryGenerator.commonServiceFilterQuery(sc);
- query.setParam("event", "/getLogLevelCounts");
+ query.setParam("event", "/service/logs/levels/counts/namevalues");
List<VNameValue> logsCounts = getLogLevelFacets(query);
nameValueList.setVNameValues(logsCounts);
@@ -628,40 +628,40 @@ public class LogsMgr extends MgrBase {
public List<Count> getFacetCounts(SolrQuery solrQuery, String facetField)
throws SolrServerException, IOException, SolrException {
List<Count> list = new ArrayList<FacetField.Count>();
-
+
QueryResponse response = serviceLogsSolrDao.process(solrQuery);
if(response == null){
return list;
}
-
+
FacetField field = response.getFacetField(facetField);
if (field == null) {
return list;
}
list = field.getValues();
-
-
+
+
return list;
}
public String getPageByKeyword(SearchCriteria searchCriteria)
throws SolrServerException {
String defaultChoice = "0";
-
+
String key = (String) searchCriteria.getParamValue("keyword");
if(stringUtil.isEmpty(key)){
throw restErrorUtil.createRESTException("Keyword was not given",
MessageEnums.DATA_NOT_FOUND);
}
-
+
String keyword = solrUtil.escapeForStandardTokenizer(key);
-
+
if(keyword.startsWith("\"") && keyword.endsWith("\"")){
keyword = keyword.substring(1);
keyword = keyword.substring(0, keyword.length()-1);
}
keyword = "*" + keyword + "*";
-
+
String keyType = (String) searchCriteria.getParamValue("keywordType");
QueryResponse queryResponse = null;
@@ -684,7 +684,7 @@ public class LogsMgr extends MgrBase {
nextPageLogTimeQuery.remove("rows");
nextPageLogTimeQuery.setStart(lastLogIndexNumber);
nextPageLogTimeQuery.setRows(1);
-
+
queryResponse = serviceLogsSolrDao.process(
nextPageLogTimeQuery);
if(queryResponse == null){
@@ -697,7 +697,7 @@ public class LogsMgr extends MgrBase {
throw restErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found",
MessageEnums.ERROR_SYSTEM);
}
-
+
SolrDocument solrDoc = docList.get(0);
Date logDate = (Date) solrDoc.get(LogSearchConstants.LOGTIME);
@@ -765,7 +765,7 @@ public class LogsMgr extends MgrBase {
if (!stringUtil.isEmpty(sortByType) && sortByType
.equalsIgnoreCase(LogSearchConstants.ASCENDING_ORDER)) {
-
+
queryGenerator.setSingleRangeFilter(logTimeThroughRangeQuery,
LogSearchConstants.LOGTIME, nextPageLogTime,
endTime);
@@ -774,7 +774,7 @@ public class LogsMgr extends MgrBase {
+ LogSearchConstants.ASCENDING_ORDER);
} else {
-
+
queryGenerator.setSingleRangeFilter(logTimeThroughRangeQuery,
LogSearchConstants.LOGTIME, startTime,
nextPageLogTime);
@@ -803,7 +803,7 @@ public class LogsMgr extends MgrBase {
if (!documentList.isEmpty()){
solrDocument = documentList.get(0);
}
-
+
Date keywordLogDate = (Date) solrDocument.get(LogSearchConstants.LOGTIME);
if(keywordLogDate == null){
throw restErrorUtil.createRESTException("The keyword "+"\""+key+"\""+" was not found",
@@ -837,9 +837,9 @@ public class LogsMgr extends MgrBase {
long countNumberLogs = countQuery(rangeLogQuery,serviceLogsSolrDao) - 1;
-
- //Adding numbers on
+
+ //Adding numbers on
try {
@@ -854,7 +854,7 @@ public class LogsMgr extends MgrBase {
String id = (String) solrDocumenent
.getFieldValue(LogSearchConstants.ID);
countNumberLogs++;
-
+
if (stringUtil.isEmpty(id) && id.equals(keywordId)){
break;
}
@@ -980,8 +980,8 @@ public class LogsMgr extends MgrBase {
logTimeThroughRangeQuery.set(LogSearchConstants.SORT,
LogSearchConstants.LOGTIME + " "
+ LogSearchConstants.DESCENDING_ORDER);
-
-
+
+
queryGenerator.setSingleRangeFilter(
logTimeThroughRangeQuery,
LogSearchConstants.LOGTIME, startTime,
@@ -993,7 +993,7 @@ public class LogsMgr extends MgrBase {
logTimeThroughRangeQuery.set(LogSearchConstants.SORT,
LogSearchConstants.LOGTIME + " "
+ LogSearchConstants.ASCENDING_ORDER);
-
+
queryGenerator.setSingleRangeFilter(logTimeThroughRangeQuery,
LogSearchConstants.LOGTIME, lastLogsLogTime, endTime);
@@ -1054,7 +1054,7 @@ public class LogsMgr extends MgrBase {
long countNumberLogs = countQuery(rangeLogQuery,serviceLogsSolrDao) - 1;
-
+
//Adding numbers on
try {
SolrQuery sameIdQuery = queryGenerator
@@ -1123,9 +1123,9 @@ public class LogsMgr extends MgrBase {
.process(logTimeByIdQuery);
if(queryResponse == null){
- return convertObjToString(new VSolrLogList());
+ return convertObjToString(new VSolrLogList());
}
-
+
SolrDocumentList docList = queryResponse.getResults();
Date dateOfLogId = null;
if (docList != null && !docList.isEmpty()) {
@@ -1162,10 +1162,10 @@ public class LogsMgr extends MgrBase {
queryGenerator.setSingleIncludeFilter(sameIdQuery,
LogSearchConstants.LOGTIME, "\"" + logTime + "\"");
sameIdQuery.set("fl", LogSearchConstants.ID);
-
+
QueryResponse sameIdResponse = serviceLogsSolrDao.process(sameIdQuery);
SolrDocumentList docList = sameIdResponse.getResults();
-
+
for (SolrDocument solrDocumenent : docList) {
String id = (String) solrDocumenent
.getFieldValue(LogSearchConstants.ID);
@@ -1216,14 +1216,14 @@ public class LogsMgr extends MgrBase {
return logsCounts;
}
-
+
@SuppressWarnings("rawtypes")
RangeFacet rangeFacet=rangeFacetList.get(0);
if (rangeFacet == null) {
return logsCounts;
}
logLevelCounts = rangeFacet.getCounts();
-
+
if(logLevelCounts == null){
return logsCounts;
}
@@ -1254,7 +1254,7 @@ public class LogsMgr extends MgrBase {
String deafalutValue = "0";
VBarDataList dataList = new VBarDataList();
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
- solrQuery.set("event", "/getHistogramData");
+ solrQuery.set("event", "/audit/logs/histogram");
String from = getFrom((String) searchCriteria.getParamValue("from"));
String to = getTo((String) searchCriteria.getParamValue("to"));
String unit = getUnit((String) searchCriteria.getParamValue("unit"));
@@ -1339,10 +1339,7 @@ public class LogsMgr extends MgrBase {
}
}
- public String cancelFindRequestByDate(HttpServletRequest request) {
- String uniqueId = null;
-
- uniqueId = (String) request.getParameter("token");
+ public String cancelFindRequestByDate(String uniqueId) {
if (stringUtil.isEmpty(uniqueId)) {
logger.error("Unique id is Empty");
throw restErrorUtil.createRESTException("Unique id is Empty",
@@ -1377,21 +1374,21 @@ public class LogsMgr extends MgrBase {
String to = (String) searchCriteria.getParamValue("to");
String utcOffset = (String) searchCriteria.getParamValue("utcOffset");
String format = (String) searchCriteria.getParamValue("format");
-
+
format = defaultFormat.equalsIgnoreCase(format) && format != null ? ".txt"
: ".json";
-
+
if(stringUtil.isEmpty(utcOffset)){
utcOffset = "0";
}
-
+
if (!dateUtil.isDateValid(from) || !dateUtil.isDateValid(to)) {
logger.error("Not valid date format. Valid format should be"
+ LogSearchConstants.SOLR_DATE_FORMAT_PREFIX_Z);
throw restErrorUtil.createRESTException("Not valid date format. Valid format should be"
+ LogSearchConstants.SOLR_DATE_FORMAT_PREFIX_Z,
MessageEnums.INVALID_INPUT_DATA);
-
+
} else {
from = from.replace("T", " ");
from = from.replace(".", ",");
@@ -1411,7 +1408,7 @@ public class LogsMgr extends MgrBase {
fileName = searchCriteria.getParamValue("hostLogFile") + "_"
+ searchCriteria.getParamValue("compLogFile");
}
-
+
String textToSave = "";
try {
QueryResponse response = serviceLogsSolrDao.process(solrQuery);
@@ -1516,7 +1513,7 @@ public class LogsMgr extends MgrBase {
public String getComponentListWithLevelCounts(SearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
- solrQuery.setParam("event", "/getComponentListWithLevelCounts");
+ solrQuery.setParam("event", "/service/logs/components/level/counts");
if (searchCriteria.getSortBy() == null) {
searchCriteria.setSortBy(LogSearchConstants.SOLR_COMPONENT);
@@ -1586,10 +1583,10 @@ public class LogsMgr extends MgrBase {
queryGenerator.setMainQuery(solrQuery, null);
solrQuery.setSort(LogSearchConstants.LOGTIME, SolrQuery.ORDER.asc);
queryGenerator.setRowCount(solrQuery, 1);
-
+
List<VNameValue> vNameValues = new ArrayList<VNameValue>();
QueryResponse response = serviceLogsSolrDao.process(solrQuery);
-
+
if(response == null){
return convertObjToString(nameValueList);
}
@@ -1635,7 +1632,7 @@ public class LogsMgr extends MgrBase {
}
}
nameValueList.setVNameValues(vNameValues);
-
+
} catch (SolrServerException | SolrException | IOException e) {
logger.error(e.getMessage() + "SolrQuery"+solrQuery);
@@ -1789,7 +1786,7 @@ public class LogsMgr extends MgrBase {
if(stringUtil.isEmpty(scrollType)){
scrollType = "";
}
-
+
String logTime = null;
String sequenceId = null;
try {
@@ -1827,7 +1824,7 @@ public class LogsMgr extends MgrBase {
}
vSolrLogList.setSolrDocuments(solrDocList);
return convertObjToString(vSolrLogList);
-
+
} else if (LogSearchConstants.SCROLL_TYPE_AFTER.equals(scrollType)) {
SolrDocumentList solrDocList = new SolrDocumentList();
vSolrLogList = new VSolrLogList();
@@ -1859,9 +1856,9 @@ public class LogsMgr extends MgrBase {
}
vSolrLogList.setSolrDocuments(initial);
-
+
return convertObjToString(vSolrLogList);
-
+
}
}
@@ -1964,4 +1961,4 @@ public class LogsMgr extends MgrBase {
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsREST.java
new file mode 100644
index 0000000..3d99dc0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsREST.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.rest;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.manager.AuditMgr;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.*;
+
+@Api(value = "audit/logs", description = "Audit log operations")
+@Path("audit/logs")
+@Component
+@Scope("request")
+public class AuditLogsREST {
+
+ @Autowired
+ AuditMgr auditMgr;
+
+ @GET
+ @Path("/schema/fields")
+ @Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_SCHEMA_FIELD_LIST_OD)
+ public String getSolrFieldList(@Context HttpServletRequest request) {
+ return auditMgr.getAuditLogsSchemaFieldsName();
+ }
+
+ @GET
+ @Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_LOGS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = IS_LAST_PAGE_D, name = "isLastPage", dataType = "boolean", paramType = "query")
+ })
+ public String getAuditLogs(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ searchCriteria.addParam("isLastPage", request.getParameter("isLastPage"));
+ return auditMgr.getLogs(searchCriteria);
+ }
+
+ @GET
+ @Path("/components")
+ @Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_COMPONENTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ })
+ public String getAuditComponents(@Context HttpServletRequest request) {
+
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addParam("q", request.getParameter("q"));
+ return auditMgr.getAuditComponents(searchCriteria);
+ }
+
+ @GET
+ @Path("/linegraph")
+ @Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_LINE_GRAPH_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
+ public String getAuditLineGraphData(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ searchCriteria.addParam("unit", request.getParameter("unit"));
+ return auditMgr.getAuditLineGraphData(searchCriteria);
+ }
+
+ @GET
+ @Path("/users")
+ @Produces({"application/json"})
+ @ApiOperation(GET_TOP_AUDIT_USERS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query")
+ })
+ public String getTopAuditUsers(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ searchCriteria.addParam("field", request.getParameter("field"));
+ return auditMgr.topTenUsers(searchCriteria);
+ }
+
+ @GET
+ @Path("/resources")
+ @Produces({"application/json"})
+ @ApiOperation(GET_TOP_AUDIT_RESOURCES_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query")
+ })
+ public String getTopAuditResources(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ searchCriteria.addParam("field", request.getParameter("field"));
+ //return auditMgr.getTopAuditFieldCount(searchCriteria);
+ return auditMgr.topTenResources(searchCriteria);
+
+ }
+
+ @GET
+ @Path("/live/count")
+ @Produces({"application/json"})
+ @ApiOperation(GET_LIVE_LOGS_COUNT_OD)
+ public String getLiveLogsCount() {
+ return auditMgr.getLiveLogCounts();
+ }
+
+ @GET
+ @Path("/request/user/linegraph")
+ @Produces({"application/json"})
+ @ApiOperation(GET_REQUEST_USER_LINE_GRAPH_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
+ public String getRequestUserLineGraph(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ searchCriteria.addParam("field", request.getParameter("field"));
+ searchCriteria.addParam("unit", request.getParameter("unit"));
+ return auditMgr.getRequestUserLineGraph(searchCriteria);
+ }
+
+ @GET
+ @Path("/anygraph")
+ @Produces({"application/json"})
+ @ApiOperation(GET_ANY_GRAPH_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = X_AXIS_D, name = "xAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = Y_AXIS_D, name = "yAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = STACK_BY_D, name = "stackBy", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
+ public String getAnyGraphData(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addParam("xAxis", request.getParameter("xAxis"));
+ searchCriteria.addParam("yAxis", request.getParameter("yAxis"));
+ searchCriteria.addParam("stackBy", request.getParameter("stackBy"));
+ searchCriteria.addParam("from", request.getParameter("from"));
+ searchCriteria.addParam("to", request.getParameter("to"));
+ searchCriteria.addParam("unit", request.getParameter("unit"));
+ return auditMgr.getAnyGraphData(searchCriteria);
+ }
+
+ @GET
+ @Path("/users/export")
+ @Produces({"application/json"})
+ @ApiOperation(EXPORT_USER_TALBE_TO_TEXT_FILE_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FORMAT_D, name = "format", dataType = "string", paramType = "query")
+ })
+ public Response exportUserTableToTextFile(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ searchCriteria.addParam("field", request.getParameter("field"));
+ searchCriteria.addParam("format", request.getParameter("format"));
+ return auditMgr.exportUserTableToTextFile(searchCriteria);
+ }
+
+ @GET
+ @Path("/serviceload")
+ @Produces({"application/json"})
+ @ApiOperation(GET_SERVICE_LOAD_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ })
+ public String getServiceLoad(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredAuditLogsParams(request);
+ return auditMgr.getServiceLoad(searchCriteria);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
deleted file mode 100644
index 5ed49fd..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logsearch.rest;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.ambari.logsearch.common.SearchCriteria;
-import org.apache.ambari.logsearch.manager.AuditMgr;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.*;
-import static org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.*;
-
-@Api(value = "audit", description = "Audit operations")
-@Path("audit")
-@Component
-@Scope("request")
-public class AuditREST {
-
- @Autowired
- AuditMgr auditMgr;
-
- @GET
- @Path("/getAuditSchemaFieldsName")
- @Produces({"application/json"})
- @ApiOperation(GET_AUDIT_SCHEMA_FIELD_LIST_OD)
- public String getSolrFieldList(@Context HttpServletRequest request) {
- return auditMgr.getAuditLogsSchemaFieldsName();
- }
-
- @GET
- @Path("/getAuditLogs")
- @Produces({"application/json"})
- @ApiOperation(GET_AUDIT_LOGS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = IS_LAST_PAGE_D, name = "isLastPage", dataType = "boolean", paramType = "query")
- })
- public String getAuditLogs(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("isLastPage", request.getParameter("isLastPage"));
- return auditMgr.getLogs(searchCriteria);
- }
-
- @GET
- @Path("/getAuditComponents")
- @Produces({"application/json"})
- @ApiOperation(GET_AUDIT_COMPONENTS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- })
- public String getAuditComponents(@Context HttpServletRequest request) {
-
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addParam("q", request.getParameter("q"));
- return auditMgr.getAuditComponents(searchCriteria);
- }
-
- @GET
- @Path("/getAuditLineGraphData")
- @Produces({"application/json"})
- @ApiOperation(GET_AUDIT_LINE_GRAPH_DATA_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
- })
- public String getAuditLineGraphData(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("unit", request.getParameter("unit"));
- return auditMgr.getAuditLineGraphData(searchCriteria);
- }
-
- @GET
- @Path("/getTopAuditUsers")
- @Produces({"application/json"})
- @ApiOperation(GET_TOP_AUDIT_USERS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query")
- })
- public String getTopAuditUsers(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("field", request.getParameter("field"));
- return auditMgr.topTenUsers(searchCriteria);
- }
-
- @GET
- @Path("/getTopAuditResources")
- @Produces({"application/json"})
- @ApiOperation(GET_TOP_AUDIT_RESOURCES_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query")
- })
- public String getTopAuditResources(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("field", request.getParameter("field"));
- //return auditMgr.getTopAuditFieldCount(searchCriteria);
- return auditMgr.topTenResources(searchCriteria);
-
-
- }
-
- @GET
- @Path("/getTopAuditComponents")
- @Produces({"application/json"})
- @ApiOperation(GET_TOP_AUDIT_COMPONENTS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
- })
- public String getTopAuditComponents(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("field", request.getParameter("field"));
- searchCriteria.addParam("unit", request.getParameter("unit"));
- return auditMgr.getTopAuditFieldCount(searchCriteria);
- }
-
- @GET
- @Path("/getLiveLogsCount")
- @Produces({"application/json"})
- @ApiOperation(GET_LIVE_LOGS_COUNT_OD)
- public String getLiveLogsCount() {
- return auditMgr.getLiveLogCounts();
- }
-
- @GET
- @Path("/getRequestUserLineGraph")
- @Produces({"application/json"})
- @ApiOperation(GET_REQUEST_USER_LINE_GRAPH_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
- })
- public String getRequestUserLineGraph(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("field", request.getParameter("field"));
- searchCriteria.addParam("unit", request.getParameter("unit"));
- return auditMgr.getRequestUserLineGraph(searchCriteria);
- }
-
- @GET
- @Path("/getAnyGraphData")
- @Produces({"application/json"})
- @ApiOperation(GET_ANY_GRAPH_DATA_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = X_AXIS_D, name = "xAxis", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = Y_AXIS_D, name = "yAxis", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = STACK_BY_D, name = "stackBy", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
- })
- public String getAnyGraphData(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addParam("xAxis", request.getParameter("xAxis"));
- searchCriteria.addParam("yAxis", request.getParameter("yAxis"));
- searchCriteria.addParam("stackBy", request.getParameter("stackBy"));
- searchCriteria.addParam("from", request.getParameter("from"));
- searchCriteria.addParam("to", request.getParameter("to"));
- searchCriteria.addParam("unit", request.getParameter("unit"));
- return auditMgr.getAnyGraphData(searchCriteria);
- }
-
- @GET
- @Path("/exportUserTableToTextFile")
- @Produces({"application/json"})
- @ApiOperation(EXPORT_USER_TALBE_TO_TEXT_FILE_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FORMAT_D, name = "format", dataType = "string", paramType = "query")
- })
- public Response exportUserTableToTextFile(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- searchCriteria.addParam("field", request.getParameter("field"));
- searchCriteria.addParam("format", request.getParameter("format"));
- return auditMgr.exportUserTableToTextFile(searchCriteria);
- }
-
- @GET
- @Path("/getServiceLoad")
- @Produces({"application/json"})
- @ApiOperation(GET_SERVICE_LOAD_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- })
- public String getServiceLoad(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredAuditLogsParams(request);
- return auditMgr.getServiceLoad(searchCriteria);
- }
-
-}
-
\ No newline at end of file
[07/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json
new file mode 100644
index 0000000..bc167ba
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.0_to_2.2.2.json
@@ -0,0 +1,408 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.0",
+ "target-version": "2.2.2",
+ "options": {
+ "config-types": {
+ "capacity-scheduler": {
+ "merged-copy": "yes"
+ },
+ "cluster-env": {
+ "merged-copy": "yes"
+ },
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "hadoop-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-log4j": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "hive-env": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-env": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "oozie-env": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+ "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.default-node-label-expression": " "
+ },
+ "core-site": {
+ "hadoop.http.authentication.simple.anonymous.allowed": "true"
+ },
+ "hadoop-env": {
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options
appended to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGC
DateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The f
ollowing applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# ex
port HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /us
r/share/java/*mysql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n if [ -d \"/etc/tez/conf/\" ]; then\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n fi\nfi\n\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION
$HADOOP_OPTS\""
+ },
+ "hbase-env": {
+ "content": "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to
enable JMX exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG
_DIR={{log_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport HBASE_RE
GIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {
+ "remove": "yes"
+ }
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop
.root.logger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.
appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.secur
ity.logger}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesyste
m.audit=false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoo
p.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metri
cs.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN"
+ },
+ "hdfs-site": {
+ "dfs.datanode.max.transfer.threads": "16384",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600"
+ },
+ "hive-env": {
+ "content": "\n if [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra libraries required f
or hive compilation/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\nelif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog\nfi\n\nexport METASTORE_PORT={{hive_metastore_port}}"
+ },
+ "hive-site": {
+ "datanucleus.cache.level2.type": "none",
+ "hive.auto.convert.sortmerge.join.noconditionaltask": {
+ "remove": "yes"
+ },
+ "fs.file.impl.disable.cache": {
+ "remove": "yes"
+ },
+ "fs.hdfs.impl.disable.cache": {
+ "remove": "yes"
+ },
+ "hive.auto.convert.join.noconditionaltask.size": "238026752",
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.compactor.abortedtxn.threshold": "1000",
+ "hive.compactor.check.interval": "300L",
+ "hive.compactor.delta.num.threshold": "10",
+ "hive.compactor.delta.pct.threshold": "0.1f",
+ "hive.compactor.initiator.on": "false",
+ "hive.compactor.worker.threads": "0",
+ "hive.compactor.worker.timeout": "86400L",
+ "hive.compute.query.using.stats": "true",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.heapsize": {
+ "remove": "yes"
+ },
+ "hive.enforce.sortmergebucketmapjoin": "true",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.failure.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.post.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.pre.hooks": "org.apache.hadoop.hive.ql.hooks.ATSHook",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.execution.engine": "mr",
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.limit.optimize.enable": "true",
+ "hive.limit.pushdown.memory.usage": "0.04",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.kerberos.keytab.file": "/etc/security/keytabs/hive.service.keytab",
+ "hive.metastore.kerberos.principal": "hive/_HOST@EXAMPLE.COM",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.reducededuplication.min.reducer": "4",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.orc.splits.include.file.footer": "false",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.authentication.spnego.keytab": "HTTP/_HOST@EXAMPLE.COM",
+ "hive.server2.authentication.spnego.principal": "/etc/security/keytabs/spnego.service.keytab",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.tez.default.queues": "default",
+ "hive.server2.tez.initialize.default.sessions": "false",
+ "hive.server2.tez.sessions.per.default.queue": "1",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.transport.mode": "binary",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.autogather": "true",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.container.size": "682",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.input.format": "org.apache.hadoop.hive.ql.io.HiveInputFormat",
+ "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager",
+ "hive.txn.max.open.batch": "1000",
+ "hive.txn.timeout": "300",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.enabled": "true",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "hive.vectorized.groupby.flush.percent": "0.1",
+ "hive.vectorized.groupby.maxentries": "100000",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.enable.impersonation": {
+ "remove": "yes"
+ },
+ "hive.optimize.mapjoin.mapreduce": {
+ "remove": "yes"
+ }
+ },
+ "hiveserver2-site": {
+ "hive.security.authenticator.manager": "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
+ },
+ "mapred-env": {
+ "content": "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n#export HADOOP_JOB_HISTORYSERVER_OPTS=\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\""
+ },
+ "mapred-site": {
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}"
+ },
+ "oozie-env": {
+ "content": "\n#!/bin/bash\n\nif [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\nfi\n\n#Set JAVA HOME\nexport JAVA_HOME={{java_home}}\n\nexport JRE_HOME=${JAVA_HOME}\n\n# Set Oozie specific environment variables here.\n\n# Settings for the Embedded Tomcat that runs Oozie\n# Java System properties for Oozie should be specified in this variable\n#\n# export CATALINA_OPTS=\n\n# Oozie configuration file to load from Oozie configuration directory\n#\n# export OOZIE_CONFIG_FILE=oozie-site.xml\n\n# Oozie logs directory\n#\nexport OOZIE_LOG={{oozie_log_dir}}\n\n# Oozie pid directory\n#\nexport CATALINA_PID={{pid_file}}\n\n#Location of the data for oozie\nexport OOZIE_DATA={{oozie_data_dir}}\n\n# Oozie Log4J configuration file to load from Oozie confi
guration directory\n#\n# export OOZIE_LOG4J_FILE=oozie-log4j.properties\n\n# Reload interval of the Log4J configuration file, in seconds\n#\n# export OOZIE_LOG4J_RELOAD=10\n\n# The port Oozie server runs\n#\nexport OOZIE_HTTP_PORT={{oozie_server_port}}\n\n# The admin port Oozie server runs\n#\nexport OOZIE_ADMIN_PORT={{oozie_server_admin_port}}\n\n# The host name Oozie server runs on\n#\n# export OOZIE_HTTP_HOSTNAME=`hostname -f`\n\n# The base URL for callback URLs to Oozie\n#\n# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\nexport JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64\n\n# At least 1 minute of retry time to account for server downtime during\n# upgrade/downgrade\nexport OOZIE_CLIENT_OPTS=\"${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 \"\n\n# This is needed so that Oozie does not run into OOM or GC Overhead limit\n# exceeded exceptions. If the oozie server is handling large number of\n# workflows/coordinator jobs, th
e memory settings may need to be revised\nexport CATALINA_OPTS=\"${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m \""
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.HadoopAccessorService.kerberos.enabled": "false",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.services": "\n org.apache.oozie.service.SchedulerService,\n org.apache.oozie.service.InstrumentationService,\n org.apache.oozie.service.MemoryLocksService,\n org.apache.oozie.service.UUIDService,\n org.apache.oozie.service.ELService,\n org.apache.oozie.service.AuthorizationService,\n org.apache.oozie.service.UserGroupInformationService,\n org.apache.oozie.service.HadoopAccessorService,\n org.apache.oozie.service.JobsConcurrencyService,\n org.apache.oozie.service.URIHandlerService,\n org.apache.oozie.service.DagXLogInfoService,\n org.apache.oozie.service.SchemaService,\n org.apache.oozie.service.LiteWorkflowAppService,\n org.apache.oozie.service.JPAService,\n org.apache.oozie.service.StoreService,\n org.apache.oozie.service.CoordinatorStoreService,\n org.apache.oozie.service.SLAStoreService,\n org.apache.oozie.service.DBLiteWorkflowStoreService,\n org.apache.oozie
.service.CallbackService,\n org.apache.oozie.service.ShareLibService,\n org.apache.oozie.service.CallableQueueService,\n org.apache.oozie.service.ActionService,\n org.apache.oozie.service.ActionCheckerService,\n org.apache.oozie.service.RecoveryService,\n org.apache.oozie.service.PurgeService,\n org.apache.oozie.service.CoordinatorEngineService,\n org.apache.oozie.service.BundleEngineService,\n org.apache.oozie.service.DagEngineService,\n org.apache.oozie.service.CoordMaterializeTriggerService,\n org.apache.oozie.service.StatusTransitService,\n org.apache.oozie.service.PauseTransitService,\n org.apache.oozie.service.GroupsService,\n org.apache.oozie.service.ProxyUserService,\n org.apache.oozie.service.XLogStreamingService,\n org.apache.oozie.service.JvmPauseMonitorService",
+ "oozie.services.ext": "org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService"
+ },
+ "ranger-hbase-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hbase",
+ "REPOSITORY_CONFIG_USERNAME": "hbase",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "ranger-hbase-plugin-enabled": "No"
+ },
+ "ranger-hdfs-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hadoop",
+ "REPOSITORY_CONFIG_USERNAME": "hadoop",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "hadoop.rpc.protection": "-",
+ "ranger-hdfs-plugin-enabled": "No"
+ },
+ "ranger-hive-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hive",
+ "REPOSITORY_CONFIG_USERNAME": "hive",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver",
+ "ranger-hive-plugin-enabled": "No"
+ },
+ "webhcat-site": {
+ "templeton.hadoop": "/usr/hdp/current/hadoop-client/bin/hadoop",
+ "templeton.hcat": "/usr/hdp/current/hive-client/bin/hcat",
+ "templeton.hive.archive": "hdfs:///hdp/apps/${hdp.version}/hive/hive.tar.gz",
+ "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+ "templeton.libjars": "/usr/hdp/current/zookeeper-client/zookeeper.jar",
+ "templeton.pig.archive": "hdfs:///hdp/apps/${hdp.version}/pig/pig.tar.gz",
+ "templeton.sqoop.archive": "hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop.tar.gz",
+ "templeton.sqoop.home": "sqoop.tar.gz/sqoop",
+ "templeton.sqoop.path": "sqoop.tar.gz/sqoop/bin/sqoop",
+ "templeton.streaming.jar": "hdfs:///hdp/apps/${hdp.version}/mapreduce/hadoop-streaming.jar"
+ },
+ "yarn-site": {
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "yarn.client.nodemanager-connect.max-wait-ms": "60000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-per-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "{{yarn_log_dir_prefix}}/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "true",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "true",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.generic-application-history.store-class": "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.path": "/hadoop/yarn/timeline",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms": "300000",
+ "yarn.timeline-service.store-class": "org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore",
+ "yarn.timeline-service.ttl-ms": "2678400000"
+ }
+ }
+ }
+ ]
+}
[33/55] [abbrv] ambari git commit: AMBARI-18165. Alert on Atlas after
adding it to a secure cluster as HBase table initialization fails. (smohanty
via yusaku)
Posted by ol...@apache.org.
AMBARI-18165. Alert on Atlas after adding it to a secure cluster as HBase table initialization fails. (smohanty via yusaku)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c370449
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c370449
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c370449
Branch: refs/heads/branch-dev-logsearch
Commit: 4c37044902b57ed674e00ee93cba3e100d81518c
Parents: df1b77b
Author: Yusaku Sako <yu...@hortonworks.com>
Authored: Fri Aug 19 16:11:31 2016 -0700
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Fri Aug 19 16:11:31 2016 -0700
----------------------------------------------------------------------
.../package/templates/atlas_hbase_setup.rb.j2 | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4c370449/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
index 70d0844..14167dc 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
@@ -1,3 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
_tbl_titan = '{{atlas_graph_storage_hbase_table}}'
_tbl_audit = '{{atlas_audit_hbase_tablename}}'
_usr_atlas = '{{metadata_user}}'
[29/55] [abbrv] ambari git commit: Storm components were stopped
during RU and can not be started (Dmitry Lysnichenko via ncole)
Posted by ol...@apache.org.
Storm components were stopped during RU and can not be started (Dmitry Lysnichenko via ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ae47ae96
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ae47ae96
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ae47ae96
Branch: refs/heads/branch-dev-logsearch
Commit: ae47ae96c70c4fbc672d1539a07999a2819ea998
Parents: c17d980
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Aug 19 16:58:23 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Aug 19 16:58:23 2016 -0400
----------------------------------------------------------------------
.../StormUpgradeKerberosDescriptorConfig.java | 144 +++++++++++++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 4 +
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 2 +
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 4 +
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 2 +
5 files changed, 156 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ae47ae96/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/StormUpgradeKerberosDescriptorConfig.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/StormUpgradeKerberosDescriptorConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/StormUpgradeKerberosDescriptorConfig.java
new file mode 100644
index 0000000..07bdaa6
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/StormUpgradeKerberosDescriptorConfig.java
@@ -0,0 +1,144 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ <p>
+ http://www.apache.org/licenses/LICENSE-2.0
+ <p>
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.orm.dao.ArtifactDAO;
+import org.apache.ambari.server.orm.entities.ArtifactEntity;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
+import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+
+import com.google.inject.Inject;
+
+
+/**
+ * Update Kerberos Descriptor Storm properties when upgrading to Storm 1.0
+ */
+public class StormUpgradeKerberosDescriptorConfig extends AbstractServerAction {
+
+
+ public static final String KERBEROS_DESCRIPTOR = "kerberos_descriptor";
+ public static final String STORM = "STORM";
+ public static final String NEW_PART = "org.apache";
+ public static final String OLD_PART = "backtype";
+
+ @Inject
+ ArtifactDAO artifactDAO;
+
+ protected StormUpgradeKerberosDescriptorConfig() {
+ }
+
+ /**
+ * Lists config types and properties that may be replaced "backtype" to "org.apache"
+ */
+ private static final HashMap<String, String[]> TARGET_PROPERTIES = new HashMap<String, String[]>() {{
+ put("storm-site", new String[]{
+ "nimbus.authorizer",
+ "storm.principal.tolocal",
+ "drpc.authorizer"
+ });
+
+ }};
+
+ /**
+ * Update Kerberos Descriptor Storm properties when upgrading to Storm 1.0
+ * <p/>
+ * Finds the relevant artifact entities and iterates through them to process each independently.
+ */
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+ String msg = "";
+
+ List<ArtifactEntity> artifactEntities = artifactDAO.findByName(KERBEROS_DESCRIPTOR);
+
+ if (artifactEntities != null) {
+ for (ArtifactEntity artifactEntity : artifactEntities) {
+ if (artifactEntity == null) {
+ continue;
+ }
+ msg = updateKerberosDescriptorConfigurations(msg, artifactEntity);
+ }
+ } else {
+ msg = msg + MessageFormat.format("{0} not found\n", KERBEROS_DESCRIPTOR);
+ }
+
+ msg = MessageFormat.format("{0}\n Successfully replace properties", msg);
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", msg, "");
+ }
+
+ private String updateKerberosDescriptorConfigurations(String msg, ArtifactEntity artifactEntity) {
+ Map<String, Object> data = artifactEntity.getArtifactData();
+
+ if (data != null) {
+ final KerberosDescriptor kerberosDescriptor = new KerberosDescriptorFactory().createInstance(data);
+
+ if (kerberosDescriptor != null) {
+ // Get the service that needs to be updated
+ KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(STORM);
+
+ if (serviceDescriptor != null) {
+ for (String configType : TARGET_PROPERTIES.keySet()) {
+ // Get the configuration that needs to be updated
+ KerberosConfigurationDescriptor configurationDescriptor = serviceDescriptor.getConfiguration(configType);
+
+ if (configurationDescriptor != null) {
+ for (String key : TARGET_PROPERTIES.get(configType)) {
+ // Get the configuration properties that needs to be updated
+ String value = configurationDescriptor.getProperty(key);
+ if (value != null) {
+ value = value.replace(OLD_PART, NEW_PART);
+ configurationDescriptor.putProperty(key, value);
+ msg = msg + MessageFormat.format("{0}={1}, \n", key, value);
+ } else {
+ msg = msg + MessageFormat.format("{0} not found in {1}\n", key, configType);
+ }
+
+ artifactEntity.setArtifactData(kerberosDescriptor.toMap());
+ artifactDAO.merge(artifactEntity);
+ }
+ } else {
+ msg = msg + MessageFormat.format("{0} not found\n", configType);
+ }
+ }
+ } else {
+ msg = msg + MessageFormat.format("{0} not found\n", STORM);
+ }
+ artifactEntity.setArtifactData(kerberosDescriptor.toMap());
+ artifactDAO.merge(artifactEntity);
+ } else {
+ msg = msg + "KerberosDescriptor not created from data";
+ }
+ } else {
+ msg = msg + MessageFormat.format("{0} not found\n", "artifactData");
+ }
+ return msg;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ae47ae96/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 65a7880..b9067c7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -427,6 +427,10 @@
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus in KerberosDescriptor">
+ <task xsi:type="server_action" summary="Upgrade Storm Security Configs to 1.0"
+ class="org.apache.ambari.server.serveraction.upgrades.StormUpgradeKerberosDescriptorConfig"/>
+ </execute-stage>
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Storm">
<!-- Remove Atlas configs that were incorrectly added to storm-site instead of Atlas' application.properties. -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/ae47ae96/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index a256d2f..7237a6a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -985,6 +985,8 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_storm_audit_db" />
<task xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0"/>
+ <task xsi:type="server_action" summary="Upgrade Storm Security Configs to 1.0"
+ class="org.apache.ambari.server.serveraction.upgrades.StormUpgradeKerberosDescriptorConfig"/>
<!-- Remove Atlas configs that were incorrectly added to storm-site instead of Atlas' application.properties. -->
<task xsi:type="configure" id="hdp_2_5_0_0_remove_storm_atlas_configs" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/ae47ae96/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 20ddfbb..6f9372e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -373,6 +373,10 @@
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus in KerberosDescriptor">
+ <task xsi:type="server_action" summary="Upgrade Storm Security Configs to 1.0"
+ class="org.apache.ambari.server.serveraction.upgrades.StormUpgradeKerberosDescriptorConfig"/>
+ </execute-stage>
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Storm">
<!-- Remove Atlas configs that were incorrectly added to storm-site instead of Atlas' application.properties. -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/ae47ae96/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index a4eb513..f20ad06 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -954,6 +954,8 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_storm_audit_db" />
<task xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0"/>
+ <task xsi:type="server_action" summary="Upgrade Storm Security Configs to 1.0"
+ class="org.apache.ambari.server.serveraction.upgrades.StormUpgradeKerberosDescriptorConfig"/>
<!-- Remove Atlas configs that were incorrectly added to storm-site instead of Atlas' application.properties. -->
<task xsi:type="configure" id="hdp_2_5_0_0_remove_storm_atlas_configs" />
[45/55] [abbrv] ambari git commit: AMBARI-18196. Generate REST API
docs with Swagger for Log Search (oleewere)
Posted by ol...@apache.org.
AMBARI-18196. Generate REST API docs with Swagger for Log Search (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/650e6c4e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/650e6c4e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/650e6c4e
Branch: refs/heads/branch-dev-logsearch
Commit: 650e6c4e91d65874996732e9c81111a8bcaa1153
Parents: 1743180
Author: oleewere <ol...@gmail.com>
Authored: Thu Aug 18 12:15:52 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:02 2016 +0200
----------------------------------------------------------------------
ambari-logsearch/ambari-logsearch-it/pom.xml | 1 +
.../ambari-logsearch-portal/pom.xml | 32 +++
.../org/apache/ambari/logsearch/LogSearch.java | 39 ++-
.../ambari/logsearch/doc/DocConstants.java | 135 +++++++++
.../apache/ambari/logsearch/rest/AuditREST.java | 145 ++++++++++
.../ambari/logsearch/rest/DashboardREST.java | 285 +++++++++++++++++++
.../ambari/logsearch/rest/LogFileREST.java | 20 ++
.../ambari/logsearch/rest/PublicREST.java | 6 +
.../ambari/logsearch/rest/UserConfigREST.java | 21 ++
.../main/webapp/META-INF/applicationContext.xml | 16 ++
.../META-INF/security-applicationContext.xml | 3 +-
.../src/main/webapp/WEB-INF/web.xml | 5 +-
.../src/main/webapp/login.html | 2 +-
.../src/main/webapp/scripts/App.js | 4 +-
.../src/main/webapp/scripts/utils/Globals.js | 4 +-
.../scripts/views/audit/AuditAggregatedView.js | 2 +-
.../webapp/scripts/views/tabs/LogFileView.js | 2 +-
.../main/webapp/templates/helpers/Helpers.js | 4 +-
18 files changed, 705 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-it/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/pom.xml b/ambari-logsearch/ambari-logsearch-it/pom.xml
index 075a22a..d918c8a 100644
--- a/ambari-logsearch/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-it/pom.xml
@@ -113,6 +113,7 @@
<include>**/*Stories.java</include>
<include>**/*Story.java</include>
</includes>
+ <skip>${it.skip}</skip>
</configuration>
</execution>
<execution>
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 7a46fe5..349531e 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -35,6 +35,7 @@
<spring.ldap.version>2.0.4.RELEASE</spring.ldap.version>
<jersey.version>1.19</jersey.version>
<jetty-version>9.2.11.v20150529</jetty-version>
+ <swagger.version>1.5.8</swagger.version>
</properties>
<profiles>
<!-- Dev Profile Start -->
@@ -646,5 +647,36 @@
<artifactId>cglib</artifactId>
<version>3.2.4</version>
</dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ <version>${swagger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-core</artifactId>
+ <version>${swagger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-jaxrs</artifactId>
+ <version>${swagger.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-models</artifactId>
+ <version>${swagger.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.webjars</groupId>
+ <artifactId>swagger-ui</artifactId>
+ <version>2.1.0</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
index 819d3b9..d1fbcde 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logsearch;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.ServerSocket;
import java.net.URI;
import java.net.URISyntaxException;
@@ -38,6 +39,9 @@ import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.handler.HandlerList;
+import org.eclipse.jetty.server.handler.ResourceHandler;
+import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.webapp.WebAppContext;
@@ -79,14 +83,13 @@ public class LogSearch {
public void run(String[] argv) throws Exception {
Server server = buildSever(argv);
- URI webResourceBase = findWebResourceBase(server.getClass()
- .getClassLoader());
- WebAppContext context = new WebAppContext();
- context.setBaseResource(Resource.newResource(webResourceBase));
- context.setContextPath(ROOT_CONTEXT);
- context.setParentLoaderPriority(true);
- server.setHandler(context);
+ HandlerList handlers = new HandlerList();
+ handlers.addHandler(createSwaggerContext());
+ handlers.addHandler(createBaseWebappContext());
+
+ server.setHandler(handlers);
server.start();
+
logger
.debug("============================Server Dump=======================================");
logger.debug(server.dump());
@@ -95,7 +98,7 @@ public class LogSearch {
ConfigUtil.initializeApplicationConfig();
server.join();
}
-
+
public Server buildSever(String argv[]) {
Server server = new Server();
ServerConnector connector = new ServerConnector(server);
@@ -145,6 +148,26 @@ public class LogSearch {
return server;
}
+ private WebAppContext createBaseWebappContext() throws MalformedURLException {
+ URI webResourceBase = findWebResourceBase(LogSearch.class.getClassLoader());
+ WebAppContext context = new WebAppContext();
+ context.setBaseResource(Resource.newResource(webResourceBase));
+ context.setContextPath(ROOT_CONTEXT);
+ context.setParentLoaderPriority(true);
+ return context;
+ }
+
+ private ServletContextHandler createSwaggerContext() throws URISyntaxException {
+ ResourceHandler resourceHandler = new ResourceHandler();
+ resourceHandler.setResourceBase(LogSearch.class.getClassLoader()
+ .getResource("META-INF/resources/webjars/swagger-ui/2.1.0")
+ .toURI().toString());
+ ServletContextHandler context = new ServletContextHandler();
+ context.setContextPath("/docs/");
+ context.setHandler(resourceHandler);
+ return context;
+ }
+
private URI findWebResourceBase(ClassLoader classLoader) {
URL fileCompleteUrl = Thread.currentThread().getContextClassLoader()
.getResource(WEB_RESOURCE_FOLDER);
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
new file mode 100644
index 0000000..c1572b7
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.doc;
+
+public class DocConstants {
+
+ public class CommonDescriptions {
+ public static final String X_AXIS_D = "";
+ public static final String Y_AXIS_D = "";
+ public static final String STACK_BY_D = "";
+ public static final String EXCLUDE_QUERY_D = "";
+ public static final String INCLUDE_QUERY_D = "";
+ public static final String MUST_BE_D = "";
+ public static final String MUST_NOT_D = "";
+ public static final String FROM_D = "";
+ public static final String TO_D = "";
+ public static final String UNIT_D = "";
+ public static final String QUERY_D = "";
+ public static final String COLUMN_QUERY_D = "";
+ public static final String I_MESSAGE_D = "";
+ public static final String G_E_MESSAGE_D = "";
+ public static final String E_MESSAGE_D = "";
+ public static final String IS_LAST_PAGE_D = "";
+ public static final String FIELD_D = "";
+ public static final String FORMAT_D = "";
+ }
+
+ public class AuditOperationDescriptions {
+ public static final String GET_AUDIT_SCHEMA_FIELD_LIST_OD = "";
+ public static final String GET_AUDIT_LOGS_OD = "";
+ public static final String GET_AUDIT_COMPONENTS_OD = "";
+ public static final String GET_AUDIT_LINE_GRAPH_DATA_OD = "";
+ public static final String GET_TOP_AUDIT_USERS_OD = "";
+ public static final String GET_TOP_AUDIT_RESOURCES_OD = "";
+ public static final String GET_TOP_AUDIT_COMPONENTS_OD = "";
+ public static final String GET_LIVE_LOGS_COUNT_OD = "";
+ public static final String GET_REQUEST_USER_LINE_GRAPH_OD = "";
+ public static final String GET_ANY_GRAPH_DATA_OD = "";
+ public static final String EXPORT_USER_TALBE_TO_TEXT_FILE_OD = "";
+ public static final String GET_SERVICE_LOAD_OD = "";
+ }
+
+ public class ServiceDescriptions {
+ public static final String LEVEL_D = "";
+ public static final String ADVANCED_SEARCH_D = "";
+ public static final String TREE_PARAMS_D = "";
+ public static final String START_TIME_D = "";
+ public static final String END_TIME_D = "";
+ public static final String FILE_NAME_D = "";
+ public static final String HOST_NAME_D = "";
+ public static final String COMPONENT_NAME_D = "";
+ public static final String FIND_D = "";
+ public static final String ID_D = "";
+ public static final String HOST_D = "";
+ public static final String COMPONENT_D = "";
+ public static final String KEYWORD_TYPE_D = "";
+ public static final String TOKEN_D = "";
+ public static final String SOURCE_LOG_ID_D = "";
+ public static final String G_MUST_NOT_D = "";
+ public static final String NUMBER_ROWS_D = "";
+ public static final String SCROLL_TYPE_D = "";
+ public static final String UTC_OFFSET_D = "";
+ }
+
+ public class ServiceOperationDescriptions {
+ public static final String SEARCH_LOGS_OD = "";
+ public static final String GET_HOSTS_OD = "";
+ public static final String GET_COMPONENTS_OD = "";
+ public static final String GET_AGGREGATED_INFO_OD = "";
+ public static final String GET_LOG_LEVELS_COUNT_OD = "";
+ public static final String GET_COMPONENTS_COUNT_OD = "";
+ public static final String GET_HOSTS_COUNT_OD = "";
+ public static final String GET_TREE_EXTENSION_OD = "";
+ public static final String GET_HISTOGRAM_DATA_OD = "";
+ public static final String CANCEL_FIND_REQUEST_OD = "";
+ public static final String EXPORT_TO_TEXT_FILE_OD = "";
+ public static final String GET_COMPONENT_LIST_WITH_LEVEL_COUNT_OD = "";
+ public static final String GET_EXTREME_DATES_FOR_BUNDLE_ID_OD = "";
+ public static final String GET_SERVICE_LOGS_FIELD_NAME_OD = "";
+ public static final String GET_ANY_GRAPH_DATA_OD = "";
+ public static final String GET_AFTER_BEFORE_LOGS_OD = "";
+ public static final String GET_HOST_LIST_BY_COMPONENT_OD = "";
+ public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "";
+ public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "";
+ }
+
+
+ public class LogFileDescriptions {
+ public static final String HOST_D = "";
+ public static final String COMPONENT_D = "";
+ public static final String LOG_TYPE_D = "";
+ }
+
+ public class LogFileOperationDescriptions {
+ public static final String SEARCH_LOG_FILES_OD = "";
+ public static final String GET_LOG_FILE_TAIL_OD = "";
+ }
+
+ public class PublicOperationDescriptions {
+ public static final String OBTAIN_GENERAL_CONFIG_OD = "Obtain general config";
+ }
+
+ public class UserConfigDescriptions {
+ public static final String USER_ID_D = "";
+ public static final String FILTER_NAME_D = "";
+ public static final String ROW_TYPE_D = "";
+ }
+
+ public class UserConfigOperationDescriptions {
+ public static final String SAVE_USER_CONFIG_OD = "Save user config";
+ public static final String UPDATE_USER_CONFIG_OD = "Update user config";
+ public static final String DELETE_USER_CONFIG_OD = "Delete user config";
+ public static final String GET_USER_CONFIG_OD = "Get user config";
+ public static final String GET_USER_FILTER_OD = "Get user filter";
+ public static final String UPDATE_USER_FILTER_OD = "Update user filter";
+ public static final String GET_USER_FILTER_BY_ID_OD = "Get user filter by id";
+ public static final String GET_ALL_USER_NAMES_OD = "Get all user names";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
index 6d18b4a..5ed49fd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java
@@ -26,12 +26,20 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.common.SearchCriteria;
import org.apache.ambari.logsearch.manager.AuditMgr;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.*;
+
+@Api(value = "audit", description = "Audit operations")
@Path("audit")
@Component
@Scope("request")
@@ -43,6 +51,7 @@ public class AuditREST {
@GET
@Path("/getAuditSchemaFieldsName")
@Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_SCHEMA_FIELD_LIST_OD)
public String getSolrFieldList(@Context HttpServletRequest request) {
return auditMgr.getAuditLogsSchemaFieldsName();
}
@@ -50,6 +59,21 @@ public class AuditREST {
@GET
@Path("/getAuditLogs")
@Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_LOGS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = IS_LAST_PAGE_D, name = "isLastPage", dataType = "boolean", paramType = "query")
+ })
public String getAuditLogs(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -60,6 +84,10 @@ public class AuditREST {
@GET
@Path("/getAuditComponents")
@Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_COMPONENTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ })
public String getAuditComponents(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
@@ -70,6 +98,21 @@ public class AuditREST {
@GET
@Path("/getAuditLineGraphData")
@Produces({"application/json"})
+ @ApiOperation(GET_AUDIT_LINE_GRAPH_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
public String getAuditLineGraphData(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -80,6 +123,21 @@ public class AuditREST {
@GET
@Path("/getTopAuditUsers")
@Produces({"application/json"})
+ @ApiOperation(GET_TOP_AUDIT_USERS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query")
+ })
public String getTopAuditUsers(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -90,6 +148,21 @@ public class AuditREST {
@GET
@Path("/getTopAuditResources")
@Produces({"application/json"})
+ @ApiOperation(GET_TOP_AUDIT_RESOURCES_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query")
+ })
public String getTopAuditResources(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -103,6 +176,22 @@ public class AuditREST {
@GET
@Path("/getTopAuditComponents")
@Produces({"application/json"})
+ @ApiOperation(GET_TOP_AUDIT_COMPONENTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
public String getTopAuditComponents(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -114,6 +203,7 @@ public class AuditREST {
@GET
@Path("/getLiveLogsCount")
@Produces({"application/json"})
+ @ApiOperation(GET_LIVE_LOGS_COUNT_OD)
public String getLiveLogsCount() {
return auditMgr.getLiveLogCounts();
}
@@ -121,6 +211,22 @@ public class AuditREST {
@GET
@Path("/getRequestUserLineGraph")
@Produces({"application/json"})
+ @ApiOperation(GET_REQUEST_USER_LINE_GRAPH_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
public String getRequestUserLineGraph(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -132,6 +238,15 @@ public class AuditREST {
@GET
@Path("/getAnyGraphData")
@Produces({"application/json"})
+ @ApiOperation(GET_ANY_GRAPH_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = X_AXIS_D, name = "xAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = Y_AXIS_D, name = "yAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = STACK_BY_D, name = "stackBy", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query")
+ })
public String getAnyGraphData(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addParam("xAxis", request.getParameter("xAxis"));
@@ -146,6 +261,22 @@ public class AuditREST {
@GET
@Path("/exportUserTableToTextFile")
@Produces({"application/json"})
+ @ApiOperation(EXPORT_USER_TALBE_TO_TEXT_FILE_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIELD_D, name = "field", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FORMAT_D, name = "format", dataType = "string", paramType = "query")
+ })
public Response exportUserTableToTextFile(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
@@ -157,6 +288,20 @@ public class AuditREST {
@GET
@Path("/getServiceLoad")
@Produces({"application/json"})
+ @ApiOperation(GET_SERVICE_LOAD_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ })
public String getServiceLoad(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredAuditLogsParams(request);
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
index cb87f6f..0144edc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
@@ -25,6 +25,10 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.SearchCriteria;
import org.apache.ambari.logsearch.manager.LogsMgr;
@@ -37,6 +41,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.ServiceDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.ServiceOperationDescriptions.*;
+
+@Api(value = "dashboard", description = "Dashboard operations")
@Path("dashboard")
@Component
@Scope("request")
@@ -51,6 +60,34 @@ public class DashboardREST {
@GET
@Path("/solr/logs_search")
@Produces({"application/json"})
+ @ApiOperation(SEARCH_LOGS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIND_D, name = "find", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = SOURCE_LOG_ID_D, name = "sourceLogId", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = KEYWORD_TYPE_D, name = "keywordType", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TOKEN_D, name = "token", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = IS_LAST_PAGE_D, name = "isLastPage", dataType = "boolean", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String searchSolrData(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -70,6 +107,10 @@ public class DashboardREST {
@GET
@Path("/hosts")
@Produces({"application/json"})
+ @ApiOperation(GET_HOSTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query")
+ })
public String getHosts(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addParam("q", request.getParameter("q"));
@@ -79,6 +120,10 @@ public class DashboardREST {
@GET
@Path("/components")
@Produces({"application/json"})
+ @ApiOperation(GET_COMPONENTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query")
+ })
public String getComponents(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addParam("q", request.getParameter("q"));
@@ -88,6 +133,27 @@ public class DashboardREST {
@GET
@Path("/aggregatedData")
@Produces({"application/json"})
+ @ApiOperation(GET_AGGREGATED_INFO_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String getAggregatedInfo(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria();
searchCriteria.addRequiredServiceLogsParams(request);
@@ -97,6 +163,12 @@ public class DashboardREST {
@GET
@Path("/levels_count")
@Produces({"application/json"})
+ @ApiOperation(GET_LOG_LEVELS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public VCountList getLogLevelsCount(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria();
searchCriteria.addParam("q", request.getParameter("q"));
@@ -109,6 +181,12 @@ public class DashboardREST {
@GET
@Path("/components_count")
@Produces({"application/json"})
+ @ApiOperation(GET_COMPONENTS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public VCountList getComponentsCount(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria();
searchCriteria.addParam("q", request.getParameter("q"));
@@ -121,6 +199,14 @@ public class DashboardREST {
@GET
@Path("/hosts_count")
@Produces({"application/json"})
+ @ApiOperation(GET_HOSTS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query")
+ })
public VCountList getHostsCount(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria();
searchCriteria.addParam("q", request.getParameter("q"));
@@ -137,6 +223,29 @@ public class DashboardREST {
@GET
@Path("/getTreeExtension")
@Produces({"application/json"})
+ @ApiOperation(GET_TREE_EXTENSION_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public VNodeList getTreeExtension(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -150,6 +259,29 @@ public class DashboardREST {
@GET
@Path("/getLogLevelCounts")
@Produces({"application/json"})
+ @ApiOperation(GET_LOG_LEVELS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public VNameValueList getLogsLevelCount(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -162,6 +294,30 @@ public class DashboardREST {
@GET
@Path("/getHistogramData")
@Produces({"application/json"})
+ @ApiOperation(GET_HISTOGRAM_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String getHistogramData(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -175,6 +331,7 @@ public class DashboardREST {
@GET
@Path("/cancelFindRequest")
@Produces({"application/json"})
+ @ApiOperation(CANCEL_FIND_REQUEST_OD)
public String cancelFindRequest(@Context HttpServletRequest request) {
return logMgr.cancelFindRequestByDate(request);
}
@@ -182,6 +339,31 @@ public class DashboardREST {
@GET
@Path("/exportToTextFile")
@Produces({"application/json"})
+ @ApiOperation(EXPORT_TO_TEXT_FILE_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FORMAT_D, name = "format", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UTC_OFFSET_D, name = "utcOffset", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public Response exportToTextFile(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
@@ -198,6 +380,29 @@ public class DashboardREST {
@GET
@Path("/getHostListByComponent")
@Produces({"application/json"})
+ @ApiOperation(GET_HOST_LIST_BY_COMPONENT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String getHostListByComponent(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -212,6 +417,29 @@ public class DashboardREST {
@GET
@Path("/getComponentListWithLevelCounts")
@Produces({"application/json"})
+ @ApiOperation(GET_COMPONENT_LIST_WITH_LEVEL_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String getComponentListWithLevelCounts(
@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
@@ -225,6 +453,7 @@ public class DashboardREST {
@GET
@Path("/solr/getBundleIdBoundaryDates")
@Produces({"application/json"})
+ @ApiOperation(GET_EXTREME_DATES_FOR_BUNDLE_ID_OD)
public String getExtremeDatesForBundelId(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
@@ -238,6 +467,7 @@ public class DashboardREST {
@GET
@Path("/getServiceLogsFieldsName")
@Produces({"application/json"})
+ @ApiOperation(GET_SERVICE_LOGS_FIELD_NAME_OD)
public String getServiceLogsFieldsName() {
return logMgr.getServiceLogsFieldsName();
}
@@ -245,6 +475,7 @@ public class DashboardREST {
@GET
@Path("/getServiceLogsSchemaFieldsName")
@Produces({"application/json"})
+ @ApiOperation(GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD)
public String getServiceLogsSchemaFieldsName() {
return logMgr.getServiceLogsSchemaFieldsName();
}
@@ -252,6 +483,33 @@ public class DashboardREST {
@GET
@Path("/getAnyGraphData")
@Produces({"application/json"})
+ @ApiOperation(GET_ANY_GRAPH_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = X_AXIS_D, name = "xAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = Y_AXIS_D, name = "yAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = STACK_BY_D, name = "stackBy", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String getAnyGraphData(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -267,6 +525,32 @@ public class DashboardREST {
@GET
@Path("/getAfterBeforeLogs")
@Produces({"application/json"})
+ @ApiOperation(GET_AFTER_BEFORE_LOGS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D,name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ID_D, name = "id", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = SCROLL_TYPE_D, name = "scrollType", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = NUMBER_ROWS_D, name = "numberRows", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
public String getAfterBeforeLogs(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addRequiredServiceLogsParams(request);
@@ -284,6 +568,7 @@ public class DashboardREST {
@GET
@Path("/getHadoopServiceConfigJSON")
@Produces({"application/json"})
+ @ApiOperation(GET_HADOOP_SERVICE_CONFIG_JSON_OD)
public String getHadoopServiceConfigJSON() {
return logMgr.getHadoopServiceConfigJSON();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
index ef1bb8f..d53cff9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
@@ -24,12 +24,20 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.common.SearchCriteria;
import org.apache.ambari.logsearch.manager.LogFileMgr;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import static org.apache.ambari.logsearch.doc.DocConstants.LogFileDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.LogFileOperationDescriptions.*;
+
+@Api(value = "logfile", description = "Logfile operations")
@Path("logfile")
@Component
@Scope("request")
@@ -40,6 +48,12 @@ public class LogFileREST {
@GET
@Produces({"application/json"})
+ @ApiOperation(SEARCH_LOG_FILES_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LOG_TYPE_D, name = "logType", dataType = "string", paramType = "query")
+ })
public String searchLogFiles(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addParam("component", request.getParameter("component"));
@@ -51,6 +65,12 @@ public class LogFileREST {
@GET
@Path("/getLogFileTail")
@Produces({"application/json"})
+ @ApiOperation(GET_LOG_FILE_TAIL_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LOG_TYPE_D, name = "logType", dataType = "string", paramType = "query")
+ })
public String getLogFileTail(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria();
searchCriteria.addParam("host", request.getParameter("host"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
index 7977703..af48acd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
@@ -21,11 +21,16 @@ package org.apache.ambari.logsearch.rest;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.manager.PublicMgr;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.OBTAIN_GENERAL_CONFIG_OD;
+
+@Api(value = "public", description = "Public operations")
@Path("public")
@Component
@Scope("request")
@@ -36,6 +41,7 @@ public class PublicREST {
@GET
@Path("/getGeneralConfig")
+ @ApiOperation(OBTAIN_GENERAL_CONFIG_OD)
public String getGeneralConfig() {
return generalMgr.getGeneralConfig();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
index 40b215c..4b1675f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
@@ -29,6 +29,10 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.SearchCriteria;
import org.apache.ambari.logsearch.manager.UserConfigMgr;
@@ -38,6 +42,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+import static org.apache.ambari.logsearch.doc.DocConstants.UserConfigDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.UserConfigOperationDescriptions.*;
+
+@Api(value = "userconfig", description = "User config operations")
@Path("userconfig")
@Component
@Scope("request")
@@ -52,6 +60,7 @@ public class UserConfigREST {
@POST
@Path("/saveUserConfig")
@Produces({"application/json"})
+ @ApiOperation(SAVE_USER_CONFIG_OD)
public String saveUserConfig(VUserConfig vhist) {
return userConfigMgr.saveUserConfig(vhist);
}
@@ -59,12 +68,14 @@ public class UserConfigREST {
@PUT
@Path("/updateUserConfig")
@Produces({"application/json"})
+ @ApiOperation(UPDATE_USER_CONFIG_OD)
public String updateUserConfig(VUserConfig vhist) {
return userConfigMgr.updateUserConfig(vhist);
}
@DELETE
@Path("/deleteUserConfig/{id}")
+ @ApiOperation(DELETE_USER_CONFIG_OD)
public void deleteUserConfig(@PathParam("id") String id) {
userConfigMgr.deleteUserConfig(id);
}
@@ -72,6 +83,12 @@ public class UserConfigREST {
@GET
@Path("/getUserConfig")
@Produces({"application/json"})
+ @ApiOperation(GET_USER_CONFIG_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = USER_ID_D, name = "userId", paramType = "query", dataType = "string"),
+ @ApiImplicitParam(value = FILTER_NAME_D, name = "filterName", paramType = "query", dataType = "string"),
+ @ApiImplicitParam(value = ROW_TYPE_D, name = "rowType", paramType = "query", dataType = "string")
+ })
public String getUserConfig(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria(request);
searchCriteria.addParam(LogSearchConstants.USER_NAME,
@@ -86,6 +103,7 @@ public class UserConfigREST {
@GET
@Path("/user_filter")
@Produces({"application/json"})
+ @ApiOperation(GET_USER_FILTER_OD)
public String getUserFilter(@Context HttpServletRequest request) {
return userConfigMgr.getUserFilter();
}
@@ -93,6 +111,7 @@ public class UserConfigREST {
@POST
@Path("/user_filter")
@Produces({"application/json"})
+ @ApiOperation(UPDATE_USER_FILTER_OD)
public String createUserFilter(String json) {
return userConfigMgr.saveUserFiter(json);
}
@@ -100,6 +119,7 @@ public class UserConfigREST {
@PUT
@Path("/user_filter/{id}")
@Produces({"application/json"})
+ @ApiOperation(GET_USER_FILTER_BY_ID_OD)
public String updateUserFilter(String json) {
return userConfigMgr.saveUserFiter(json);
}
@@ -107,6 +127,7 @@ public class UserConfigREST {
@GET
@Path("/getAllUserName")
@Produces({"application/json"})
+ @ApiOperation(GET_ALL_USER_NAMES_OD)
public String getAllUserName() {
return userConfigMgr.getAllUserName();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml
index 5e24d88..f334e67 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml
@@ -49,5 +49,21 @@ http://www.springframework.org/schema/util/spring-util.xsd">
</property>
<property name="propertiesPersister" ref="xmlPropertyConfigurer" />
</bean>
+
+ <bean id="host" class="java.net.InetAddress" factory-method="getLocalHost"/>
+ <bean id="apiListingResource" class="io.swagger.jaxrs.listing.ApiListingResource"/>
+ <bean id="swaggerSerializers" class="io.swagger.jaxrs.listing.SwaggerSerializers" scope="singleton"/>
+ <bean id="beanConfig" class="io.swagger.jaxrs.config.BeanConfig">
+ <property name="schemes" value="http" /> <!-- TODO: set this from property -->
+ <property name="resourcePackage" value="org.apache.ambari.logsearch.rest"/>
+ <property name="version" value="1.0.0"/>
+ <property name="host" value="#{host.hostAddress}:61888"/> <!-- TODO: set port from property -->
+ <property name="basePath" value="/api/v1"/>
+ <property name="title" value="Log Search REST API"/>
+ <property name="description" value="Log aggregation, analysis, and visualization."/>
+ <property name="license" value="Apache 2.0"/>
+ <property name="licenseUrl" value="http://www.apache.org/licenses/LICENSE-2.0.html"/>
+ <property name="scan" value="true"/>
+ </bean>
</beans>
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
index 96852b5..233fad4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
@@ -31,8 +31,9 @@ http://www.springframework.org/schema/security/spring-security-4.0.xsd">
<security:http pattern="/libs/**" security="none" />
<security:http pattern="/images/**" security="none" />
<security:http pattern="/templates/**" security="none" />
- <security:http pattern="/service/public/**" security="none" />
<security:http pattern="/favicon.ico" security="none" />
+ <security:http pattern="/api/v1/public/**" security="none" />
+ <security:http pattern="/api/v1/swagger.json" security="none"/>
<security:http disable-url-rewriting="true" use-expressions="true" create-session="always" entry-point-ref="authenticationProcessingFilterEntryPoint">
<csrf disabled="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
index dc8519a..aee16bf 100755
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
@@ -65,8 +65,7 @@
</servlet>
<servlet-mapping>
<servlet-name>REST service</servlet-name>
- <url-pattern>/service/*</url-pattern>
+ <url-pattern>/api/v1/*</url-pattern>
</servlet-mapping>
-
-
+
</web-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
index ff546ef..44f1aeb 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
@@ -35,7 +35,7 @@
var passDiv = $('#passCont');
$.ajax({
- url : "/service/public/getGeneralConfig",
+ url : "/api/v1/public/getGeneralConfig",
type: "GET",
async: true,
dataType: 'json',
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/App.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/App.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/App.js
index ef9b662..24cbd5a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/App.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/App.js
@@ -60,7 +60,7 @@ define(['backbone','utils/LangSupport', 'backbone.marionette'],function(Backbone
context: this,
url: tempUrl,
success: function(response){
- serverUrl += "http://" + response.parameters['logsearch.server.url']+"/service/";
+ serverUrl += "http://" + response.parameters['logsearch.server.url']+"/api/v1/";
},
error: function(response){
console.log("Error while getting log search server url");
@@ -70,7 +70,7 @@ define(['backbone','utils/LangSupport', 'backbone.marionette'],function(Backbone
App.baseUrl = serverUrl//Utils.getLogSearchHostDetails();
} else {
- App.baseUrl = "service/";
+ App.baseUrl = "api/v1/";
}
// Add initialize hooks
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Globals.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Globals.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Globals.js
index 212846b..b6e8d7d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Globals.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Globals.js
@@ -39,8 +39,8 @@ define(['require','App'], function(require, App){
totalCount : 50
};
- //Globals.baseURL = '../';
- Globals.baseURL = App.baseUrl;
+ Globals.baseURL = '../api/v1/';
+ //Globals.baseURL = App.baseUrl;
Globals.AppTabs = {
DASHBOARD : { value:1, valStr: 'Dashboard'},
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
index acca51c..c04aaf9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
@@ -408,7 +408,7 @@ define(['require',
obj.utcOffset = moment().utcOffset();
obj.startIndex = this.topUsers.state.currentPage * this.topUsers.state.pageSize;
var params = $.param(_.extend({},this.topUsers.queryParams,obj));
- var url = "service/audit/exportUserTableToTextFile?"+ params;
+ var url = "api/v1/audit/exportUserTableToTextFile?"+ params;
window.open(url);
this.onDialogClosed();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
index 6932355..4af4670 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
@@ -203,7 +203,7 @@ define(['require',
// {startIndex : this.logFileCollection.state.currentPage * this.logFileCollection.state.pageSize},obj),
// 'component','from','to','host','level','unit','startIndex','pageSize','format','utcOffset'));
var params = $.param(_.extend({}, this.logFileCollection.queryParams, obj));
- var url = "service/dashboard/exportToTextFile?" + params;
+ var url = "api/v1/dashboard/exportToTextFile?" + params;
window.open(url);
this.onDialogClosed();
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/650e6c4e/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/helpers/Helpers.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/helpers/Helpers.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/helpers/Helpers.js
index 2117aa3..f4f1d32 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/helpers/Helpers.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/helpers/Helpers.js
@@ -292,9 +292,9 @@
var path;
if(_.has(options,'profileImageGId')){
if(_.isUndefined(size))
- path = "service/content/multimedia/image/"+options.profileImageGId;
+ path = "api/v1/content/multimedia/image/"+options.profileImageGId;
else
- path = "service/content/multimedia/image/"+options.profileImageGId+"/small";
+ path = "api/v1/content/multimedia/image/"+options.profileImageGId+"/small";
}else{
path = "styles/images/s-avatar.png";
}
[55/55] [abbrv] ambari git commit: AMBARI-18225. Fixed search issue
for number field in Logsearch portal (Hayat Behlim via oleewere)
Posted by ol...@apache.org.
AMBARI-18225. Fixed search issue for number field in Logsearch portal (Hayat Behlim via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a565ead6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a565ead6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a565ead6
Branch: refs/heads/branch-dev-logsearch
Commit: a565ead6fcc7b31856af81ce0ac851aa421f7aca
Parents: d8bb21c
Author: oleewere <ol...@gmail.com>
Authored: Mon Aug 22 15:30:47 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 15:30:47 2016 +0200
----------------------------------------------------------------------
.../ambari-logsearch-portal/pom.xml | 21 ++++
.../logsearch/common/LogSearchConstants.java | 2 -
.../ambari/logsearch/dao/SolrDaoBase.java | 44 ++++----
.../apache/ambari/logsearch/dao/UserDao.java | 3 +
.../logsearch/graph/GraphDataGenerator.java | 22 ++--
.../ambari/logsearch/manager/AuditMgr.java | 6 +-
.../ambari/logsearch/manager/LogsMgr.java | 5 +-
.../ambari/logsearch/query/QueryGeneration.java | 103 ++++++++++++++-----
.../logsearch/query/QueryGenerationBase.java | 11 ++
.../ambari/logsearch/util/ConfigUtil.java | 70 +++++++------
.../apache/ambari/logsearch/util/SolrUtil.java | 54 ++++++++++
11 files changed, 243 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 349531e..a886b37 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -535,6 +535,27 @@
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>${solr.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
index 43a7eb7..5dcdac1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
@@ -91,8 +91,6 @@ public class LogSearchConstants {
public static final String UI_SUFFIX = "@UI@";
public static final String SOLR_SUFFIX = "@Solr@";
public static final String NGRAM_SUFFIX = "ngram_";
- public static final String DEFAULT_SERVICE_COLUMN_SUFFIX = "service";
- public static final String DEFAULT_AUDIT_COLUMN_SUFFIX = "audit";
//Date Format for SOLR
public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 91c4a26..cda5e26 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,10 +22,10 @@ package org.apache.ambari.logsearch.dao;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-
import org.apache.ambari.logsearch.common.LogsearchContextUtil;
import org.apache.ambari.logsearch.common.MessageEnums;
import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
@@ -60,7 +60,10 @@ import org.springframework.beans.factory.annotation.Autowired;
public abstract class SolrDaoBase {
static private Logger logger = Logger.getLogger(SolrDaoBase.class);
-
+
+ public HashMap<String, String> schemaFieldsNameMap = new HashMap<String, String>();
+ public HashMap<String, String> schemaFieldTypeMap = new HashMap<String, String>();
+
private static Logger logPerformance = Logger
.getLogger("org.apache.ambari.logsearch.performance");
@@ -95,6 +98,7 @@ public abstract class SolrDaoBase {
private boolean populateFieldsThreadActive = false;
int SETUP_RETRY_SECOND = 30;
+ int SETUP_UPDATE_SECOND = 10*60; //10 min
int ALIAS_SETUP_RETRY_SECOND = 30*60; //30 minutes
private boolean isZkConnectString=false;//by default its false
@@ -599,17 +603,16 @@ public abstract class SolrDaoBase {
}
private void populateSchemaFields() {
- boolean result = _populateSchemaFields();
- if (!result && !populateFieldsThreadActive) {
+ if (!populateFieldsThreadActive) {
populateFieldsThreadActive = true;
logger.info("Creating thread to populated fields for collection="
- + collectionName);
+ + collectionName);
Thread fieldPopulationThread = new Thread("populated_fields_"
- + collectionName) {
+ + collectionName) {
@Override
public void run() {
logger.info("Started thread to get fields for collection="
- + collectionName);
+ + collectionName);
int retryCount = 0;
while (true) {
try {
@@ -617,27 +620,25 @@ public abstract class SolrDaoBase {
retryCount++;
boolean _result = _populateSchemaFields();
if (_result) {
- logger.info("Populate fields for collection "
- + collectionName + " is success");
- break;
+ logger.info("Populate fields for collection " + collectionName
+ + " is success, Update it after " + SETUP_UPDATE_SECOND
+ + " sec");
+ Thread.sleep(SETUP_UPDATE_SECOND * 1000);
}
} catch (InterruptedException sleepInterrupted) {
- logger.info("Sleep interrupted while populating fields for collection "
- + collectionName);
+ logger
+ .info("Sleep interrupted while populating fields for collection "
+ + collectionName);
break;
} catch (Exception ex) {
logger.error("Error while populating fields for collection "
- + collectionName
- + ", retryCount="
- + retryCount);
- } finally {
- populateFieldsThreadActive = false;
+ + collectionName + ", retryCount=" + retryCount);
}
}
+ populateFieldsThreadActive = false;
logger.info("Exiting thread for populating fields. collection="
- + collectionName);
+ + collectionName);
}
-
};
fieldPopulationThread.setDaemon(true);
fieldPopulationThread.start();
@@ -650,7 +651,7 @@ public abstract class SolrDaoBase {
private boolean _populateSchemaFields() {
SolrRequest<SchemaResponse> request = new SchemaRequest();
request.setMethod(METHOD.GET);
- request.setPath("/schema/fields");
+ request.setPath("/schema");
String historyCollection = PropertiesUtil.getProperty("logsearch.solr.collection.history","history");
if (solrClient != null && !collectionName.equals(historyCollection)) {
NamedList<Object> namedList = null;
@@ -659,14 +660,13 @@ public abstract class SolrDaoBase {
logger.info("populateSchemaFields() collection="
+ collectionName + ", fields=" + namedList);
} catch (SolrException | SolrServerException | IOException e) {
-
logger.error(
"Error occured while popuplating field. collection="
+ collectionName, e);
}
if (namedList != null) {
ConfigUtil.extractSchemaFieldsName(namedList.toString(),
- collectionName);
+ schemaFieldsNameMap,schemaFieldTypeMap);
return true;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
index 091c40b..6b2f049 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
@@ -192,6 +192,9 @@ public class UserDao {
* @return
*/
public String encryptPassword(String username, String password) {
+ if (!stringUtil.isEmpty(username)) {
+ username = username.toLowerCase();
+ }
String saltEncodedpasswd = md5Encoder
.encodePassword(password, username);
return saltEncodedpasswd;
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
index 361f8e3..d3975b3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
@@ -32,6 +32,7 @@ import org.apache.ambari.logsearch.query.QueryGeneration;
import org.apache.ambari.logsearch.util.ConfigUtil;
import org.apache.ambari.logsearch.util.DateUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.ambari.logsearch.util.StringUtil;
import org.apache.ambari.logsearch.view.VBarDataList;
import org.apache.ambari.logsearch.view.VBarGraphData;
@@ -45,9 +46,12 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.TextField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+
+
@Component
public class GraphDataGenerator extends GraphDataGeneratorBase {
@@ -62,6 +66,9 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
@Autowired
DateUtil dateUtil;
+
+ @Autowired
+ SolrUtil solrUtil;
private static Logger logger = Logger.getLogger(GraphDataGenerator.class);
@@ -80,8 +87,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
String from = (String) searchCriteria.getParamValue("from");
String to = (String) searchCriteria.getParamValue("to");
String unit = (String) searchCriteria.getParamValue("unit");
- String suffix = (String) searchCriteria.getParamValue("suffix");
- String typeXAxis = ConfigUtil.schemaFieldsName.get(xAxisField + suffix);
+ String typeXAxis = solrDaoBase.schemaFieldsNameMap.get(xAxisField);
typeXAxis = (stringUtil.isEmpty(typeXAxis)) ? "string" : typeXAxis;
// Y axis credentials
@@ -102,8 +108,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, to,
solrDaoBase, typeXAxis, fieldTime, solrQuery);
case RANGE_STACK_GRAPH:
- return rangeStackGraph(xAxisField, yAxisField, stackField, from, to,
- unit, solrDaoBase, typeXAxis, fieldTime, solrQuery);
+ return rangeStackGraph(xAxisField, stackField, from, to, unit, solrDaoBase, solrQuery);
default:
logger.warn("Invalid graph type :" + garphType.name());
return null;
@@ -255,7 +260,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
queryGenerator.setMainQuery(solrQuery, mainQuery);
queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
String jsonQuery = "";
- if (isTypeNumber(typeXAxis)) {
+ if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
String function = (yAxisField.contains("count")) ? "sum" : yAxisField;
jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(stackField,
xAxisField, function);
@@ -323,7 +328,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
VBarGraphData vBarGraphData = new VBarGraphData();
Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
queryGenerator.setMainQuery(solrQuery, null);
- if (isTypeNumber(typeXAxis)) {
+ if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to);
return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase,
typeXAxis, fieldTime, solrQuery);
@@ -363,9 +368,8 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
}
@SuppressWarnings("unchecked")
- private VBarDataList rangeStackGraph(String xAxisField, String yAxisField,
- String stackField, String from, String to, String unit,
- SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime,
+ private VBarDataList rangeStackGraph(String xAxisField, String stackField,
+ String from, String to, String unit, SolrDaoBase solrDaoBase,
SolrQuery solrQuery) {
VBarDataList dataList = new VBarDataList();
List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
index 58c3a4d..ab287bc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java
@@ -468,12 +468,11 @@ public class AuditMgr extends MgrBase {
}
public String getAuditLogsSchemaFieldsName() {
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
String excludeArray[] = PropertiesUtil
.getPropertyStringList("logsearch.solr.audit.logs.exclude.columnlist");
List<String> fieldNames = new ArrayList<String>();
HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>();
- ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+ ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,auditSolrDao);
for (String fieldName : fieldNames) {
String uiField = ConfigUtil.auditLogsColumnMapping.get(fieldName
@@ -492,15 +491,12 @@ public class AuditMgr extends MgrBase {
public String getAnyGraphData(SearchCriteria searchCriteria) {
searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME);
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
- searchCriteria.addParam("suffix", suffix);
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria,
auditSolrDao, solrQuery);
if (result == null) {
result = new VBarDataList();
}
-
return convertObjToString(result);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
index 748d2f9..257f04d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
@@ -1626,12 +1626,11 @@ public class LogsMgr extends MgrBase {
public String getServiceLogsSchemaFieldsName() {
List<String> fieldNames = new ArrayList<String>();
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
String excludeArray[] = PropertiesUtil
.getPropertyStringList("logsearch.solr.service.logs.exclude.columnlist");
HashMap<String, String> uiFieldColumnMapping = new LinkedHashMap<String, String>();
- ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+ ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,serviceLogsSolrDao);
for (String fieldName : fieldNames) {
String uiField = ConfigUtil.serviceLogsColumnMapping.get(fieldName
@@ -1695,8 +1694,6 @@ public class LogsMgr extends MgrBase {
public String getAnyGraphData(SearchCriteria searchCriteria) {
searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME);
- String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
- searchCriteria.addParam("suffix", suffix);
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria,
serviceLogsSolrDao, solrQuery);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
index 646abd6..d934e84 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
@@ -27,11 +27,19 @@ import java.util.regex.Pattern;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
import org.apache.ambari.logsearch.util.ConfigUtil;
import org.apache.ambari.logsearch.util.PropertiesUtil;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
+import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.schema.TrieDoubleField;
+import org.apache.solr.schema.TrieFloatField;
+import org.apache.solr.schema.TrieLongField;
import org.springframework.stereotype.Component;
@Component
@@ -245,6 +253,10 @@ public class QueryGeneration extends QueryGenerationBase {
String value = getOriginalValue(originalKey,
"" + columnListMap.get(key));
orQuery = putWildCardByType(value, originalKey, logType);
+ if (stringUtil.isEmpty(orQuery)) {
+ logger.debug("Removing invalid filter for key :"+originalKey +" and value :" +value );
+ continue;
+ }
boolean isSame = false;
if (elments.contains(key)) {
isSame = true;
@@ -337,48 +349,72 @@ public class QueryGeneration extends QueryGenerationBase {
}
private String putWildCardByType(String str, String key, LOG_TYPE logType) {
- String type;
+ String fieldType;
+ SolrDaoBase solrDaoBase = null;
switch (logType) {
case AUDIT:
- String auditSuffix = PropertiesUtil
- .getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
- type = ConfigUtil.schemaFieldsName.get(key + auditSuffix);
+ fieldType = auditSolrDao.schemaFieldsNameMap.get(key);
+ solrDaoBase = auditSolrDao;
break;
case SERVICE:
- String serviceLogs = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
- type = ConfigUtil.schemaFieldsName.get(key + serviceLogs);
+ fieldType = serviceLogsSolrDao.schemaFieldsNameMap.get(key);
+ solrDaoBase = serviceLogsSolrDao;
if (key.equalsIgnoreCase(LogSearchConstants.SOLR_LOG_MESSAGE)) {
return solrUtil.escapeForLogMessage(key, str);
}
break;
default:
// set as null
- type = null;
+ logger.error("Invalid logtype :" + logType);
+ fieldType = null;
}
- if (type == null) {
- return key + ":" + "*" + str + "*";
- } else if ("text_std_token_lower_case".equalsIgnoreCase(type)) {
- return key + ":" + solrUtil.escapeForStandardTokenizer(str);
- } else if ("key_lower_case".equalsIgnoreCase(type)
- || "string".equalsIgnoreCase(type)) {
- return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str);
- } else if ("ip_address".equalsIgnoreCase(type)) {
- return key + ":" + str;
+ if (!stringUtil.isEmpty(fieldType)) {
+ if (solrUtil.isSolrFieldNumber(fieldType, solrDaoBase)) {
+ String value = putEscapeCharacterForNumber(str, fieldType,solrDaoBase);
+ if (!stringUtil.isEmpty(value)) {
+ return key + ":" + value;
+ } else {
+ return null;
+ }
+ } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class,solrDaoBase)) {
+ return key + ":" + solrUtil.escapeForStandardTokenizer(str);
+ } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class,solrDaoBase)|| "string".equalsIgnoreCase(fieldType)) {
+ return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str);
+ } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class,solrDaoBase)) {
+ return key + ":" + str;
+ }
}
- return key + ":" + putEscapeCharacterForNumber(str);
+ return key + ":" + "*" + str + "*";
}
- private String putEscapeCharacterForNumber(String str) {
- String escapeCharSting = "" + returnDefaultIfValueNotNumber(str);
- escapeCharSting = str.replace("-", "\\-");
+ private String putEscapeCharacterForNumber(String str,String fieldType,SolrDaoBase solrDaoBase) {
+ if (!stringUtil.isEmpty(str)) {
+ str = str.replace("*", "");
+ }
+ String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType,solrDaoBase);
+ if (escapeCharSting == null || escapeCharSting.isEmpty()) {
+ return null;
+ }
+ escapeCharSting = escapeCharSting.replace("-", "\\-");
return escapeCharSting;
}
- private String returnDefaultIfValueNotNumber(String str) {
+ private String parseInputValueAsPerFieldType(String str,String fieldType,SolrDaoBase solrDaoBase ) {
try {
- return "" + Integer.parseInt(str);
+ HashMap<String, Object> fieldTypeInfoMap= solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase);
+ String className = (String) fieldTypeInfoMap.get("class");
+ if( className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())){
+ return ""+ Double.parseDouble(str);
+ }else if(className.equalsIgnoreCase(TrieFloatField.class.getSimpleName())){
+ return ""+ Float.parseFloat(str);
+ }else if(className.equalsIgnoreCase(TrieLongField.class.getSimpleName())){
+ return ""+ Long.parseLong(str);
+ }else {
+ return "" + Integer.parseInt(str);
+ }
} catch (Exception e) {
- return "0";
+ logger.debug("Invaid input str: " + str + " For fieldType :" + fieldType);
+ return null;
}
}
@@ -436,4 +472,25 @@ public class QueryGeneration extends QueryGenerationBase {
}
return originalKey;
}
+
+ public boolean checkTokenizer(String fieldType,Class tokenizerFactoryClass,SolrDaoBase solrDaoBase) {
+ HashMap<String, Object> fieldTypeMap = solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase);
+ HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap
+ .get("analyzer");
+ if (analyzer != null) {
+ HashMap<String, Object> tokenizerMap = (HashMap<String, Object>) analyzer
+ .get("tokenizer");
+ if (tokenizerMap != null) {
+ String tokenizerClass = (String) tokenizerMap.get("class");
+ if (!StringUtils.isEmpty(tokenizerClass)) {
+ tokenizerClass =tokenizerClass.replace("solr.", "");
+ if (tokenizerClass.equalsIgnoreCase(tokenizerFactoryClass
+ .getSimpleName())) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
index a49107d..a128098 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java
@@ -24,6 +24,8 @@ import java.util.List;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.ambari.logsearch.util.QueryBase;
import org.apache.ambari.logsearch.util.SolrUtil;
@@ -34,6 +36,7 @@ import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.gson.Gson;
+import com.sun.jersey.api.spring.Autowire;
public abstract class QueryGenerationBase extends QueryBase {
@@ -47,6 +50,14 @@ public abstract class QueryGenerationBase extends QueryBase {
@Autowired
JSONUtil jsonUtil;
+
+ @Autowired
+ AuditSolrDao auditSolrDao;
+
+ @Autowired
+ ServiceLogsSolrDao serviceLogsSolrDao;
+
+
public static enum CONDITION {
OR, AND
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
index bdd304f..2661150 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java
@@ -23,9 +23,9 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.ambari.logsearch.manager.MgrBase;
import org.apache.log4j.Logger;
import org.codehaus.jettison.json.JSONArray;
@@ -38,7 +38,6 @@ public class ConfigUtil {
public static HashMap<String, String> auditLogsColumnMapping = new HashMap<String, String>();
- public static HashMap<String, String> schemaFieldsName = new HashMap<String, String>();
public static void initializeApplicationConfig() {
initializeColumnMapping();
@@ -78,57 +77,63 @@ public class ConfigUtil {
auditLogsColumnMapping);
}
+
public static void extractSchemaFieldsName(String responseString,
- String suffix) {
+ HashMap<String, String> schemaFieldsNameMap,
+ HashMap<String, String> schemaFieldTypeMap) {
try {
JSONObject jsonObject = new JSONObject(responseString);
- JSONArray jsonArrayList = jsonObject.getJSONArray("fields");
-
- if(jsonArrayList == null){
+ JSONObject schemajsonObject = jsonObject.getJSONObject("schema");
+ JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields");
+ JSONArray fieldTypeJsonArray = schemajsonObject
+ .getJSONArray("fieldTypes");
+ if (jsonArrayList == null) {
+ return;
+ }
+ if (fieldTypeJsonArray == null) {
return;
}
+ HashMap<String, String> _schemaFieldTypeMap = new HashMap<String, String>();
+ HashMap<String, String> _schemaFieldsNameMap = new HashMap<String, String>();
+ for (int i = 0; i < fieldTypeJsonArray.length(); i++) {
+ JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i);
+ String name = typeObject.getString("name");
+ String fieldTypeJson = typeObject.toString();
+ _schemaFieldTypeMap.put(name, fieldTypeJson);
+ }
for (int i = 0; i < jsonArrayList.length(); i++) {
JSONObject explrObject = jsonArrayList.getJSONObject(i);
String name = explrObject.getString("name");
String type = explrObject.getString("type");
-
if (!name.contains("@") && !name.startsWith("_")
- && !name.contains("_md5") && !name.contains("_ms")
- && !name.contains(LogSearchConstants.NGRAM_SUFFIX)
- && !name.contains("tags") && !name.contains("_str")) {
- schemaFieldsName.put(name + suffix, type);
+ && !name.contains("_md5") && !name.contains("_ms")
+ && !name.contains(LogSearchConstants.NGRAM_SUFFIX)
+ && !name.contains("tags") && !name.contains("_str")) {
+ _schemaFieldsNameMap.put(name, type);
}
}
-
+ schemaFieldsNameMap.clear();
+ schemaFieldTypeMap.clear();
+ schemaFieldsNameMap.putAll(_schemaFieldsNameMap);
+ schemaFieldTypeMap.putAll(_schemaFieldTypeMap);
} catch (Exception e) {
-
logger.error(e + "Credentials not specified in logsearch.properties "
- + MessageEnums.ERROR_SYSTEM);
-
+ + MessageEnums.ERROR_SYSTEM);
}
-
}
@SuppressWarnings("rawtypes")
- public static void getSchemaFieldsName(String suffix, String excludeArray[],
- List<String> fieldNames) {
- if (!schemaFieldsName.isEmpty()) {
- Iterator iteratorSechmaFieldsName = schemaFieldsName.entrySet()
- .iterator();
-
+ public static void getSchemaFieldsName(String excludeArray[],
+ List<String> fieldNames, SolrDaoBase solrDaoBase) {
+ if (!solrDaoBase.schemaFieldsNameMap.isEmpty()) {
+ Iterator iteratorSechmaFieldsName = solrDaoBase.schemaFieldsNameMap
+ .entrySet().iterator();
while (iteratorSechmaFieldsName.hasNext()) {
-
- Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName
- .next();
+ Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName.next();
String field = "" + fieldName.getKey();
-
- if (field.contains(suffix)) {
- field = field.replace(suffix, "");
- if (!isExclude(field, excludeArray)) {
- fieldNames.add(field);
- }
-
+ if (!isExclude(field, excludeArray)) {
+ fieldNames.add(field);
}
}
}
@@ -144,5 +149,4 @@ public class ConfigUtil {
}
return false;
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a565ead6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
index f68891f..ee706bf 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java
@@ -20,10 +20,16 @@
package org.apache.ambari.logsearch.util;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Locale;
import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.log4j.Logger;
+import org.apache.solr.schema.TrieDoubleField;
+import org.apache.solr.schema.TrieFloatField;
+import org.apache.solr.schema.TrieIntField;
+import org.apache.solr.schema.TrieLongField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -34,6 +40,9 @@ public class SolrUtil {
@Autowired
StringUtil stringUtil;
+ @Autowired
+ JSONUtil jsonUtil;
+
public String setField(String fieldName, String value) {
if (value == null || value.trim().length() == 0) {
return "";
@@ -296,4 +305,49 @@ public class SolrUtil {
return newSearch.replace(" ", "\\ ");
}
+
+
+ public boolean isSolrFieldNumber(String fieldType,SolrDaoBase solrDaoBase) {
+ if (stringUtil.isEmpty(fieldType)) {
+ return false;
+ } else {
+ HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType,solrDaoBase);
+ if (typeInfoMap == null || typeInfoMap.isEmpty()) {
+ return false;
+ }
+ String fieldTypeClassName = (String) typeInfoMap.get("class");
+ if (fieldTypeClassName.equalsIgnoreCase(TrieIntField.class
+ .getSimpleName())) {
+ return true;
+ }
+ if (fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class
+ .getSimpleName())) {
+ return true;
+ }
+ if (fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class
+ .getSimpleName())) {
+ return true;
+ }
+ if (fieldTypeClassName.equalsIgnoreCase(TrieLongField.class
+ .getSimpleName())) {
+ return true;
+ }
+ return false;
+ }
+ }
+
+ public HashMap<String, Object> getFieldTypeInfoMap(String fieldType,SolrDaoBase solrDaoBase) {
+ String fieldTypeMetaData = solrDaoBase.schemaFieldTypeMap.get(fieldType);
+ HashMap<String, Object> fieldTypeMap = jsonUtil
+ .jsonToMapObject(fieldTypeMetaData);
+ if (fieldTypeMap == null) {
+ return new HashMap<String, Object>();
+ }
+ String classname = (String) fieldTypeMap.get("class");
+ if (!stringUtil.isEmpty(classname)) {
+ classname = classname.replace("solr.", "");
+ fieldTypeMap.put("class", classname);
+ }
+ return fieldTypeMap;
+ }
}
\ No newline at end of file
[44/55] [abbrv] ambari git commit: AMBARI-18076. JBehave and docker
based Integration Test Framework for Log Search components (oleewere)
Posted by ol...@apache.org.
AMBARI-18076. JBehave and docker based Integration Test Framework for Log Search components (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/17431804
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/17431804
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/17431804
Branch: refs/heads/branch-dev-logsearch
Commit: 17431804f32e94b696f55cf24ba5cf1192eedaf4
Parents: 56fe336
Author: oleewere <ol...@gmail.com>
Authored: Tue Aug 9 15:01:52 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:02 2016 +0200
----------------------------------------------------------------------
ambari-logsearch/README.md | 11 +-
ambari-logsearch/ambari-logsearch-it/pom.xml | 130 ++++++++++
.../logsearch/domain/StoryDataRegistry.java | 110 +++++++++
.../logsearch/steps/LogSearchDockerSteps.java | 244 +++++++++++++++++++
.../ambari/logsearch/steps/SolrSteps.java | 48 ++++
.../ambari/logsearch/story/LogSearchStory.java | 58 +++++
.../logsearch/story/LogfeederParsingStory.java | 22 ++
.../src/test/resources/log4j.properties | 16 ++
.../story/logfeeder_parsing_story.story | 18 ++
ambari-logsearch/pom.xml | 2 +
10 files changed, 658 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/README.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/README.md b/ambari-logsearch/README.md
index 23790cd..5c41fcd 100644
--- a/ambari-logsearch/README.md
+++ b/ambari-logsearch/README.md
@@ -33,4 +33,13 @@ mvn -Dbuild-rpm clean package
mvn -Dbuild-deb clean package
```
3. Generated RPM/DPKG files will be found in ambari-logsearch-assembly/target folder
-
+
+## Running Integration Tests
+
+By default integration tests are not a part of the build process, you need to set ${it.skip} variable to true (docker needed here too)
+
+```bash
+# from ambari-logsearch folder
+mvn clean integration-test -Dit.skip=false
+```
+Also you can run from the IDE, but make sure all of the ambari logsearch modules are built.
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/pom.xml b/ambari-logsearch/ambari-logsearch-it/pom.xml
new file mode 100644
index 0000000..075a22a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/pom.xml
@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>ambari-logsearch</artifactId>
+ <groupId>org.apache.ambari</groupId>
+ <version>2.0.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>ambari-logsearch-it</artifactId>
+ <url>http://maven.apache.org</url>
+ <name>Ambari Logsearch Integration Test</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <it.skip>true</it.skip>
+ <jbehave.version>4.0.5</jbehave.version>
+ <jersey.version>2.23.1</jersey.version>
+ <jackson-jaxrs.version>2.6.4</jackson-jaxrs.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.jbehave</groupId>
+ <artifactId>jbehave-core</artifactId>
+ <version>${jbehave.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-logsearch-solr-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.docker-java</groupId>
+ <artifactId>docker-java</artifactId>
+ <version>3.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>${jackson-jaxrs.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>${jersey.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.7.10</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <testOutputDirectory>target/classes</testOutputDirectory>
+ <testResources>
+ <testResource>
+ <directory>src/test/java/</directory>
+ <includes>
+ <include>**/*.story</include>
+ </includes>
+ </testResource>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>run-integration-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>**/*Stories.java</include>
+ <include>**/*Story.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>verify-integration-tests</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java
new file mode 100644
index 0000000..564972a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.domain;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.core.DockerClientConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+
+public class StoryDataRegistry {
+ public static final StoryDataRegistry INSTANCE = new StoryDataRegistry();
+
+ private DockerClient dockerClient;
+ private DockerClientConfig dockerClientConfig;
+ private CloudSolrClient cloudSolrClient;
+ private boolean logsearchContainerStarted = false;
+ private String dockerHost;
+ private String ambariFolder;
+ private final int solrPort = 8886;
+ private final int logsearchPort = 61888;
+ private final int zookeeperPort = 9983;
+ private final String serviceLogsCollection = "hadoop_logs";
+ private final String auditLogsCollection = "audit_logs";
+
+ private StoryDataRegistry() {
+ }
+
+ public String getDockerHost() {
+ return dockerHost;
+ }
+
+ public void setDockerHost(String dockerHost) {
+ this.dockerHost = dockerHost;
+ }
+
+ public int getSolrPort() {
+ return solrPort;
+ }
+
+ public int getLogsearchPort() {
+ return logsearchPort;
+ }
+
+ public int getZookeeperPort() {
+ return zookeeperPort;
+ }
+
+ public DockerClient getDockerClient() {
+ return dockerClient;
+ }
+
+ public void setDockerClient(DockerClient dockerClient) {
+ this.dockerClient = dockerClient;
+ }
+
+ public String getServiceLogsCollection() {
+ return serviceLogsCollection;
+ }
+
+ public String getAuditLogsCollection() {
+ return auditLogsCollection;
+ }
+
+ public CloudSolrClient getCloudSolrClient() {
+ return cloudSolrClient;
+ }
+
+ public void setCloudSolrClient(CloudSolrClient cloudSolrClient) {
+ this.cloudSolrClient = cloudSolrClient;
+ }
+
+ public String getAmbariFolder() {
+ return ambariFolder;
+ }
+
+ public void setAmbariFolder(String ambariFolder) {
+ this.ambariFolder = ambariFolder;
+ }
+
+ public DockerClientConfig getDockerClientConfig() {
+ return dockerClientConfig;
+ }
+
+ public void setDockerClientConfig(DockerClientConfig dockerClientConfig) {
+ this.dockerClientConfig = dockerClientConfig;
+ }
+
+ public boolean isLogsearchContainerStarted() {
+ return logsearchContainerStarted;
+ }
+
+ public void setLogsearchContainerStarted(boolean logsearchContainerStarted) {
+ this.logsearchContainerStarted = logsearchContainerStarted;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchDockerSteps.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchDockerSteps.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchDockerSteps.java
new file mode 100644
index 0000000..5f8f9bf
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchDockerSteps.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.steps;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.CreateContainerResponse;
+import com.github.dockerjava.api.model.Bind;
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.api.model.ExposedPort;
+import com.github.dockerjava.api.model.Ports;
+import com.github.dockerjava.api.model.Volume;
+import com.github.dockerjava.core.DockerClientBuilder;
+import com.github.dockerjava.core.DockerClientConfig;
+import com.github.dockerjava.core.command.BuildImageResultCallback;
+import com.google.common.base.Preconditions;
+import org.apache.ambari.logsearch.domain.StoryDataRegistry;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.jbehave.core.annotations.AfterStories;
+import org.jbehave.core.annotations.BeforeStories;
+import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.When;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URL;
+import java.util.List;
+
+public class LogSearchDockerSteps {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogSearchDockerSteps.class);
+
+ @Given("logsearch docker container")
+ public void setupLogSearchContainer() throws Exception {
+ boolean logsearchStarted = StoryDataRegistry.INSTANCE.isLogsearchContainerStarted();
+ if (!logsearchStarted) {
+ DockerClient dockerClient = StoryDataRegistry.INSTANCE.getDockerClient();
+ LOG.info("Create new docker container for Log Search ..");
+ URL location = LogSearchDockerSteps.class.getProtectionDomain().getCodeSource().getLocation();
+ String ambariFolder = new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent();
+ StoryDataRegistry.INSTANCE.setAmbariFolder(ambariFolder);
+ String dockerBaseDirectory = ambariFolder + "/ambari-logsearch/docker";
+ String dockerFileLocation = dockerBaseDirectory + "/Dockerfile";
+
+ String imageId = dockerClient.buildImageCmd()
+ .withTag("ambari-logsearch:v1.0")
+ .withBaseDirectory(new File(dockerBaseDirectory))
+ .withDockerfile(new File(dockerFileLocation))
+ .exec(new BuildImageResultCallback())
+ .awaitImageId();
+ LOG.info("Docker image id: {}", imageId);
+
+ removeLogSearchContainerIfExists();
+
+ // volume bindings
+ Volume testLogsVolume = new Volume("/root/test-logs");
+ Volume testConfigVolume = new Volume("/root/test-config");
+ Volume ambariVolume = new Volume("/root/ambari");
+ Volume logfeederClassesVolume = new Volume("/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes");
+ Volume logsearchClassesVolume = new Volume("/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes");
+ Volume logsearchWebappVolume = new Volume("/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes/webapps/app");
+ Bind testLogsBind = new Bind(ambariFolder +"/ambari-logsearch/docker/test-logs", testLogsVolume);
+ Bind testConfigBind = new Bind(ambariFolder +"/ambari-logsearch/docker/test-config", testConfigVolume);
+ Bind ambariRootBind = new Bind(ambariFolder, ambariVolume);
+ Bind logfeederClassesBind = new Bind(ambariFolder + "/ambari-logsearch/ambari-logsearch-logfeeder/target/classes", logfeederClassesVolume);
+ Bind logsearchClassesBind = new Bind(ambariFolder + "/ambari-logsearch/ambari-logsearch-portal/target/classes", logsearchClassesVolume);
+ Bind logsearchWebappBind = new Bind(ambariFolder + "/ambari-logsearch/ambari-logsearch-portal/src/main/webapp", logsearchWebappVolume);
+
+ // port bindings
+ Ports ports = new Ports();
+ ports.bind(new ExposedPort(5005), new Ports.Binding("0.0.0.0", "5005"));
+ ports.bind(new ExposedPort(5006), new Ports.Binding("0.0.0.0", "5006"));
+ ports.bind(new ExposedPort(StoryDataRegistry.INSTANCE.getSolrPort()), new Ports.Binding("0.0.0.0", "8886"));
+ ports.bind(new ExposedPort(StoryDataRegistry.INSTANCE.getLogsearchPort()), new Ports.Binding("0.0.0.0", "61888"));
+ ports.bind(new ExposedPort(StoryDataRegistry.INSTANCE.getZookeeperPort()), new Ports.Binding("0.0.0.0", "9983"));
+
+ LOG.info("Creating docker cointainer...");
+ CreateContainerResponse createResponse = dockerClient.createContainerCmd("ambari-logsearch:v1.0")
+ .withHostName("logsearch.apache.org")
+ .withName("logsearch")
+ .withVolumes(testLogsVolume, testConfigVolume, ambariVolume, logfeederClassesVolume, logsearchClassesVolume, logsearchWebappVolume)
+ .withBinds(testLogsBind, testConfigBind, ambariRootBind, logfeederClassesBind, logsearchClassesBind, logsearchWebappBind)
+ .withExposedPorts(
+ new ExposedPort(StoryDataRegistry.INSTANCE.getLogsearchPort()),
+ new ExposedPort(5005),
+ new ExposedPort(5006),
+ new ExposedPort(StoryDataRegistry.INSTANCE.getSolrPort()),
+ new ExposedPort(StoryDataRegistry.INSTANCE.getZookeeperPort()))
+ .withPortBindings(ports)
+ .exec();
+ LOG.info("Created docker container id: {}", createResponse.getId());
+
+ dockerClient.startContainerCmd(createResponse.getId()).exec();
+ StoryDataRegistry.INSTANCE.setLogsearchContainerStarted(true);
+ String dockerHostFromUri = StoryDataRegistry.INSTANCE.getDockerClientConfig().getDockerHost().getHost();
+ StoryDataRegistry.INSTANCE.setDockerHost(dockerHostFromUri);
+ checkHostAndPortReachable(dockerHostFromUri, StoryDataRegistry.INSTANCE.getLogsearchPort(), "LogSearch");
+ waitUntilSolrHasAnyData();
+
+ LOG.info("Waiting for logfeeder to finish the test log parsings... (10 sec)");
+ Thread.sleep(10000);
+ }
+ }
+
+ @When("logfeeder started (parse logs & send data to solr)")
+ public void logfeederStarted() throws Exception {
+ // TODO: run ps aux to check LogFeeder process with docker exec
+ /**
+ DockerClient dockerClient = StoryDataRegistry.INSTANCE.getDockerClient();
+ ExecCreateCmdResponse execResp = dockerClient
+ .execCreateCmd(containerId)
+ .withAttachStdout(true)
+ .withCmd("ps", "aux").exec();
+ execResp.getId();
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ ExecStartResultCallback res = dockerClient
+ .execStartCmd(execResp.getId())
+ .withDetach(true)
+ .withTty(true)
+ .exec(new ExecStartResultCallback(outputStream, outputStream)).awaitCompletion();
+ **/
+ }
+
+ @BeforeStories
+ public void checkDockerApi() {
+ LOG.info("Tries to setup docker client configuration");
+ final String dockerHost = System.getenv("DOCKER_HOST");
+ final String dockerCertPath = System.getenv("DOCKER_CERT_PATH");
+ final String dockerApiVersion = System.getenv("DOCKER_API_VERSION") == null ? "1.20" : System.getenv("DOCKER_API_VERSION");
+
+ Preconditions.checkArgument(dockerHost != null, "Set 'DOCKER_HOST' env variable");
+ Preconditions.checkArgument(dockerCertPath != null, "Set 'DOCKER_CERT_PATH' env variable");
+ LOG.info("DOCKER_HOST: {}", dockerHost);
+ LOG.info("DOCKER_CERT_PATH: {}", dockerCertPath);
+ LOG.info("DOCKER_API_VERSION: {}", dockerApiVersion);
+ DockerClientConfig dockerClientConfig = DockerClientConfig.createDefaultConfigBuilder()
+ .withDockerHost(dockerHost)
+ .withDockerCertPath(dockerCertPath)
+ .withApiVersion(dockerApiVersion)
+ .withDockerTlsVerify(true)
+ .build();
+ StoryDataRegistry.INSTANCE.setDockerClientConfig(dockerClientConfig);
+ DockerClient dockerClient = DockerClientBuilder.getInstance(dockerClientConfig).build();
+ StoryDataRegistry.INSTANCE.setDockerClient(dockerClient);
+ LOG.info("Docker client setup successfully.");
+ }
+
+ @AfterStories
+ public void removeLogSearchContainer() {
+ removeLogSearchContainerIfExists();
+ }
+
+ private void removeLogSearchContainerIfExists() {
+ DockerClient dockerClient = StoryDataRegistry.INSTANCE.getDockerClient();
+ List<Container> containerList = dockerClient
+ .listContainersCmd()
+ .withShowAll(true)
+ .exec();
+
+ boolean isLogSearchContainerExists = false;
+ String containerId = null;
+ for (Container container : containerList) {
+ isLogSearchContainerExists = ArrayUtils.contains(container.getNames(), "/logsearch");
+ if (isLogSearchContainerExists) {
+ containerId = container.getId();
+ break;
+ }
+ }
+
+ if (isLogSearchContainerExists) {
+ LOG.info("Remove logsearch container: {}", containerId);
+ dockerClient.removeContainerCmd(containerId).withForce(true).exec();
+ }
+ }
+
+ private void waitUntilSolrHasAnyData() throws IOException, SolrServerException, InterruptedException {
+ boolean solrHasData = false;
+ CloudSolrClient solrClient = new CloudSolrClient(String.format("%s:%d",
+ StoryDataRegistry.INSTANCE.getDockerHost(),
+ StoryDataRegistry.INSTANCE.getZookeeperPort()));
+ StoryDataRegistry.INSTANCE.setCloudSolrClient(solrClient);
+ SolrQuery solrQuery = new SolrQuery();
+ solrQuery.setQuery("*:*");
+
+ int maxTries = 60;
+ for (int tries = 1; tries < maxTries; tries++) {
+ QueryResponse queryResponse = solrClient.query(StoryDataRegistry.INSTANCE.getServiceLogsCollection(), solrQuery);
+ SolrDocumentList list = queryResponse.getResults();
+ if (list.size() > 0) {
+ solrHasData = true;
+ break;
+ } else {
+ Thread.sleep(2000);
+ LOG.info("Solr has no data yet, retrying...");
+ }
+ }
+ if (!solrHasData) {
+ throw new IllegalStateException(String.format("Solr has no data after %d tries", maxTries));
+ }
+ }
+
+
+ private void checkHostAndPortReachable(String host, int port, String serviceName) throws InterruptedException {
+ boolean reachable = false;
+ int maxTries = 60;
+ for (int tries = 1; tries < maxTries; tries++ ) {
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(host, port), 1000);
+ reachable = true;
+ break;
+ } catch (IOException e) {
+ Thread.sleep(2000);
+ LOG.info("{} is not reachable yet, retrying..", serviceName);
+ }
+ }
+ if (!reachable) {
+ throw new IllegalStateException(String.format("%s is not reachable after %s tries", serviceName, maxTries));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
new file mode 100644
index 0000000..e1e5b99
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.steps;
+
+import org.apache.ambari.logsearch.domain.StoryDataRegistry;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.Then;
+import org.jbehave.core.annotations.When;
+import org.jbehave.core.model.Story;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+public class SolrSteps {
+
+ @Then("the number of $component docs is: $docSize")
+ public void numberOfDocsForComponent(String component, int docSize) throws IOException, SolrServerException {
+ SolrClient solrClient = StoryDataRegistry.INSTANCE.getCloudSolrClient();
+ SolrQuery solrQuery = new SolrQuery();
+ solrQuery.setQuery(String.format("type:%s", component));
+ QueryResponse queryResponse = solrClient.query(StoryDataRegistry.INSTANCE.getServiceLogsCollection(), solrQuery);
+ SolrDocumentList list = queryResponse.getResults();
+ Assert.assertEquals(docSize, list.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
new file mode 100644
index 0000000..f85eb08
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.story;
+
+import org.apache.ambari.logsearch.steps.SolrSteps;
+import org.apache.ambari.logsearch.steps.LogSearchDockerSteps;
+import org.jbehave.core.configuration.Configuration;
+import org.jbehave.core.configuration.MostUsefulConfiguration;
+import org.jbehave.core.io.LoadFromClasspath;
+import org.jbehave.core.io.StoryPathResolver;
+import org.jbehave.core.io.UnderscoredCamelCaseResolver;
+import org.jbehave.core.junit.JUnitStory;
+import org.jbehave.core.reporters.Format;
+import org.jbehave.core.reporters.StoryReporterBuilder;
+import org.jbehave.core.steps.InjectableStepsFactory;
+import org.jbehave.core.steps.InstanceStepsFactory;
+import org.junit.Test;
+
+abstract public class LogSearchStory extends JUnitStory {
+ @Override
+ public Configuration configuration() {
+ StoryPathResolver storyPathResolver = new UnderscoredCamelCaseResolver(".story");
+ return new MostUsefulConfiguration()
+ .useStoryPathResolver(storyPathResolver)
+ .useStoryLoader(new LoadFromClasspath(this.getClass()))
+ .useStoryReporterBuilder(
+ new StoryReporterBuilder().withFailureTrace(true).withDefaultFormats().withFormats(Format.CONSOLE, Format.TXT));
+ }
+
+ @Override
+ public InjectableStepsFactory stepsFactory() {
+ return new InstanceStepsFactory(configuration(),
+ new LogSearchDockerSteps(),
+ new SolrSteps());
+ }
+
+ @Test
+ public void run() throws Throwable {
+ super.run();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogfeederParsingStory.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogfeederParsingStory.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogfeederParsingStory.java
new file mode 100644
index 0000000..c502cc4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogfeederParsingStory.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.story;
+
+public class LogfeederParsingStory extends LogSearchStory {
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/log4j.properties b/ambari-logsearch/ambari-logsearch-it/src/test/resources/log4j.properties
new file mode 100644
index 0000000..956bc63
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/log4j.properties
@@ -0,0 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story b/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
new file mode 100644
index 0000000..a40d04a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
@@ -0,0 +1,18 @@
+Story Service logs are parsed and stored into Solr
+
+Narrative:
+As a user
+I want to start logsearch/logfeeder/solr components in a docker container with test logs
+So that I can parse and store the logs into Solr
+
+Scenario: Logsearch logs are stored into Solr.
+
+Given logsearch docker container
+When logfeeder started (parse logs & send data to solr)
+Then the number of logsearch_app docs is: 1
+
+Scenario: Zookeeper logs are stored into Solr.
+
+Given logsearch docker container
+When logfeeder started (parse logs & send data to solr)
+Then the number of zookeeper docs is: 3
http://git-wip-us.apache.org/repos/asf/ambari/blob/17431804/ambari-logsearch/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/pom.xml b/ambari-logsearch/pom.xml
index 0445c39..b6eb246 100644
--- a/ambari-logsearch/pom.xml
+++ b/ambari-logsearch/pom.xml
@@ -34,6 +34,7 @@
<module>ambari-logsearch-portal</module>
<module>ambari-logsearch-logfeeder</module>
<module>ambari-logsearch-solr-client</module>
+ <module>ambari-logsearch-it</module>
</modules>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -159,6 +160,7 @@
<exclude>**/*.json</exclude>
<exclude>**/*.log</exclude>
<exclude>**/*.txt</exclude>
+ <exclude>**/*.story</exclude>
<exclude>**/docker/Profile</exclude>
</excludes>
</configuration>
[28/55] [abbrv] ambari git commit: AMBARI-18165. Alert on Atlas after
adding it to a secure cluster as HBase table initialization fails (Vitalyi
Brodetskyi via alejandro)
Posted by ol...@apache.org.
AMBARI-18165. Alert on Atlas after adding it to a secure cluster as HBase table initialization fails (Vitalyi Brodetskyi via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c17d9801
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c17d9801
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c17d9801
Branch: refs/heads/branch-dev-logsearch
Commit: c17d98013cf3ae9649f85345c3dc16fae5d8e994
Parents: 4773c83
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Fri Aug 19 13:48:14 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Fri Aug 19 13:48:14 2016 -0700
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 12 +-
.../package/scripts/metadata_server.py | 20 ++-
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 14 ++
.../0.1.0.2.3/package/scripts/status_params.py | 1 +
.../package/templates/atlas_hbase_setup.rb.j2 | 24 +++
.../package/scripts/hbase_regionserver.py | 26 ---
.../HDP/2.0.6/properties/stack_features.json | 5 +
.../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 36 +++++
.../2.0.6/HBASE/test_hbase_regionserver.py | 48 ------
.../stacks/2.3/ATLAS/test_metadata_server.py | 160 ++++++++++++++++++-
.../stacks/2.5/ATLAS/test_atlas_server.py | 4 +
12 files changed, 268 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 9306a43..169fbff 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -99,3 +99,4 @@ class StackFeature:
RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change"
STORM_METRICS_APACHE_CLASSES = "storm_metrics_apache_classes"
SPARK_JAVA_OPTS_SUPPORT = "spark_java_opts_support"
+ ATLAS_HBASE_SETUP="atlas_hbase_setup"
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 7e9c0f6..2ae0c6e 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -19,7 +19,7 @@ limitations under the License.
"""
from resource_management.core.resources.system import Directory, File
-from resource_management.core.source import StaticFile, InlineTemplate
+from resource_management.core.source import StaticFile, InlineTemplate, Template
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.decorator import retry
@@ -108,6 +108,10 @@ def metadata(type='server'):
group=params.user_group
)
+ if params.security_enabled:
+ TemplateConfig(format(params.atlas_jaas_file),
+ owner=params.metadata_user)
+
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
@@ -118,9 +122,9 @@ def metadata(type='server'):
create_collection('edge_index', 'basic_configs', jaasFile)
create_collection('fulltext_index', 'basic_configs', jaasFile)
- if params.security_enabled:
- TemplateConfig(format(params.atlas_jaas_file),
- owner=params.metadata_user)
+ File(params.atlas_hbase_setup,
+ content=Template("atlas_hbase_setup.rb.j2")
+ )
def upload_conf_set(config_set, jaasFile):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index d5159b8..d585c77 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -19,7 +19,7 @@ limitations under the License.
from metadata import metadata
from resource_management.libraries.functions import conf_select, stack_select
from resource_management.core.resources.system import Execute, File
-from resource_management.libraries.script.script import Script
+from resource_management.libraries.script.script import Script, format_stack_version
from resource_management.libraries.functions.check_process_status import check_process_status
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.security_commons import build_expectations, \
@@ -69,6 +69,8 @@ class MetadataServer(Script):
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh ; {params.metadata_start_script}')
no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
+ atlas_hbase_setup_command = format("cat {atlas_hbase_setup} | hbase shell -n")
+ secure_atlas_hbase_setup_command = format("kinit -kt {hbase_user_keytab} {hbase_principal_name}; ") + atlas_hbase_setup_command
if params.stack_supports_atlas_ranger_plugin:
Logger.info('Atlas plugin is enabled, configuring Atlas plugin.')
@@ -77,6 +79,22 @@ class MetadataServer(Script):
Logger.info('Atlas plugin is not supported or enabled.')
try:
+ effective_version = format_stack_version(params.version) if upgrade_type is not None else params.stack_version_formatted
+
+ if check_stack_feature(StackFeature.ATLAS_HBASE_SETUP, effective_version):
+ if params.security_enabled and params.has_hbase_master:
+ Execute(secure_atlas_hbase_setup_command,
+ tries = 5,
+ try_sleep = 10,
+ user=params.hbase_user
+ )
+ elif params.has_ranger_admin and params.has_hbase_master and not params.security_enabled:
+ Execute(atlas_hbase_setup_command,
+ tries = 5,
+ try_sleep = 10,
+ user=params.hbase_user
+ )
+
Execute(daemon_cmd,
user=params.metadata_user,
not_if=no_op_test
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 7064de3..931a764 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -70,6 +70,7 @@ def configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled):
# server configurations
config = Script.get_config()
+exec_tmp_dir = Script.get_tmp_dir()
stack_root = Script.get_stack_root()
# Needed since this is an Atlas Hook service.
@@ -225,6 +226,19 @@ enable_ranger_atlas = False
atlas_server_xmx = default("configurations/atlas-env/atlas_server_xmx", 2048)
atlas_server_max_new_size = default("configurations/atlas-env/atlas_server_max_new_size", 614)
+hbase_master_hosts = default('/clusterHostInfo/hbase_master_hosts', [])
+has_hbase_master = not len(hbase_master_hosts) == 0
+
+ranger_admin_hosts = default('/clusterHostInfo/ranger_admin_hosts', [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+
+atlas_hbase_setup = format("{exec_tmp_dir}/atlas_hbase_setup.rb")
+atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table', None)
+atlas_audit_hbase_tablename = default('/configurations/application-properties/atlas.audit.hbase.tablename', None)
+
+hbase_user_keytab = default('/configurations/hbase-env/hbase_user_keytab', None)
+hbase_principal_name = default('/configurations/hbase-env/hbase_principal_name', None)
+
if has_ranger_admin and stack_supports_atlas_ranger_plugin:
# for create_hdfs_directory
namenode_host = set(default("/clusterHostInfo/namenode_host", []))
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index 7c52fdf..59bcbaf 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -45,6 +45,7 @@ pid_dir = default("/configurations/atlas-env/metadata_pid_dir", "/var/run/atlas"
pid_file = format("{pid_dir}/atlas.pid")
metadata_user = default("/configurations/atlas-env/metadata_user", None)
+hbase_user = default("/configurations/hbase-env/hbase_user", None)
# Security related/required params
hostname = config['hostname']
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
new file mode 100644
index 0000000..70d0844
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_hbase_setup.rb.j2
@@ -0,0 +1,24 @@
+_tbl_titan = '{{atlas_graph_storage_hbase_table}}'
+_tbl_audit = '{{atlas_audit_hbase_tablename}}'
+_usr_atlas = '{{metadata_user}}'
+
+
+if not list.include? _tbl_titan
+ begin
+ create _tbl_titan,{NAME => 'e',DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW'},{NAME => 'g',DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW'},{NAME => 'i',DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW'},{NAME => 's',DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW'},{NAME => 'm',DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW'},{NAME => 'l',DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW', TTL => 604800, KEEP_DELETED_CELLS =>false}
+ rescue RuntimeError => e
+ raise e if not e.message.include? "Table already exists"
+ end
+end
+
+
+if not list.include? _tbl_audit
+ begin
+ create _tbl_audit, {NAME => 'dt', DATA_BLOCK_ENCODING => 'FAST_DIFF', COMPRESSION =>'GZ', BLOOMFILTER =>'ROW'}
+ rescue RuntimeError => e
+ raise e if not e.message.include? "Table already exists"
+ end
+end
+
+grant _usr_atlas, 'RWCA', _tbl_titan
+grant _usr_atlas, 'RWCA', _tbl_audit
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
index 40c12d9..1a35bc7 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
@@ -90,32 +90,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
def post_start(self, env, upgrade_type=None):
import params
- self.apply_atlas_acl(params.hbase_user)
-
- def apply_atlas_acl(self, hbase_user):
- import params
-
- if params.security_enabled and params.has_atlas and params.atlas_with_managed_hbase:
- current_host = params.hostname.lower()
- sorted_rs_hosts = sorted([host.lower() for host in params.rs_hosts])
-
- # if list of rs_hosts are empty, try to apply permissions regardless of host
- if len(sorted_rs_hosts) == 0:
- can_apply_permissions = True
- else:
- can_apply_permissions = current_host == sorted_rs_hosts[0]
-
- if can_apply_permissions:
- kinit_cmd = format("{kinit_path_local} -kt {regionserver_keytab_path} {regionserver_jaas_princ}")
- permissions_cmd = [
- format("echo \"grant '{metadata_user}', 'RWXCA', '{atlas_graph_storage_hbase_table}'\" | hbase shell -n"),
- format("echo \"grant '{metadata_user}', 'RWXCA', '{atlas_audit_hbase_tablename}'\" | hbase shell -n"),
- ]
-
- # no additional logging required, as it supported by checked_call itself
- # re-tries needed to suffer fails on Kerberos wizard as RegionServer update security features status over some time
- for perm_cmd in permissions_cmd:
- shell.checked_call(format("{kinit_cmd}; {perm_cmd}"), user=params.hbase_user, tries=10, try_sleep=10)
def start(self, env, upgrade_type=None):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index a754cd5..8c838db 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -298,6 +298,11 @@
"description": "Allow Spark to generate java-opts file",
"min_version": "2.2.0.0",
"max_version": "2.4.0.0"
+ },
+ {
+ "name": "atlas_hbase_setup",
+ "description": "Use script to create Atlas tables in Hbase and set permissions for Atlas user.",
+ "min_version": "2.5.0.0"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
index 4ba59d5..84e8e46 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
@@ -22,6 +22,42 @@
<name>ATLAS</name>
<version>0.7.0.2.5</version>
<extends>common-services/ATLAS/0.7.0.2.5</extends>
+ <components>
+ <component>
+ <name>ATLAS_SERVER</name>
+ <displayName>Atlas Metadata Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HBASE/HBASE_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/metadata_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>atlas_app</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
+ </component>
+ </components>
<osSpecifics>
<osSpecific>
<osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
index a6c8e0f..a0e151f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
@@ -91,54 +91,6 @@ class TestHbaseRegionServer(RMFTestCase):
self.assert_configure_secured()
self.assertNoMoreResources()
- def test_atlas_apply_acl(self):
- config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
- with open(config_file, "r") as f:
- json_content = json.load(f)
-
- json_content['clusterHostInfo']['atlas_server_hosts'] = [
- "c6401.ambari.apache.org",
- "c6402.ambari.apache.org"
- ]
- json_content['clusterHostInfo']['hbase_rs_hosts'] = [
- "c6401.ambari.apache.org",
- "c6402.ambari.apache.org"
- ]
- json_content['configurations']['atlas-env'] = {
- "metadata_user": "atlas"
- }
- if 'hbase-site' in json_content['configurations']:
- json_content['configurations']['hbase-site']["hbase.zookeeper.quorum"] = ",".join(json_content['clusterHostInfo']['atlas_server_hosts'])
- else:
- json_content['configurations']['hbase-site'] = {
- "hbase.zookeeper.quorum": ",".join(json_content['clusterHostInfo']['atlas_server_hosts'])
- }
-
- json_content['configurations']['application-properties'] = {
- "atlas.graph.storage.hostname": json_content['configurations']['hbase-site']['hbase.zookeeper.quorum'],
- "atlas.graph.storage.hbase.table": "test"
- }
-
- mock_dicts = {}
- self.executeScript(
- self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
- classname="HbaseRegionServer",
- command="start",
- config_dict=json_content,
- stack_version=self.STACK_VERSION,
- target=RMFTestCase.TARGET_COMMON_SERVICES,
- mocks_dict=mock_dicts
- )
- permission_apply_call_found = False
- pattern_search = "/usr/bin/kinit -kt /etc/security/keytabs/hbase.service.keytab hbase/c6401.ambari.apache.org@EXAMPLE.COM; echo \"grant 'atlas', 'RWXCA', 'test'\" | hbase shell -n"
- if "checked_call" in mock_dicts:
- for _call in mock_dicts["checked_call"].call_args_list:
- if len(_call) > 0 and isinstance(_call[0], tuple) and len(_call[0]) > 0 and \
- _call[0][0] == pattern_search:
- permission_apply_call_found = True
-
- self.assertEquals(True, permission_apply_call_found)
-
def test_start_secured(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 44841f4..0f7ed2f 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -167,6 +167,149 @@ class TestMetadataServer(RMFTestCase):
self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ def configureResourcesCalledSecure(self):
+ # Both server and client
+ self.assertResourceCalled('Directory', '/etc/atlas/conf',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
+
+ # Pid dir
+ self.assertResourceCalled('Directory', '/var/run/atlas',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('Directory', '/etc/atlas/conf/solr',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755,
+ recursive_ownership = True
+ )
+ # Log dir
+ self.assertResourceCalled('Directory', '/var/log/atlas',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
+ # Data dir
+ self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0644
+ )
+ # Expanded war dir
+ self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/server/webapp',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0644
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/atlas-server/server/webapp/atlas.war',
+ content = StaticFile('/usr/hdp/current/atlas-server/server/webapp/atlas.war'),
+ )
+ host_name = u"c6401.ambari.apache.org"
+ app_props = dict(self.getConfig()['configurations']['application-properties'])
+ app_props['atlas.server.bind.address'] = host_name
+
+ metadata_protocol = "https" if app_props["atlas.enableTLS"] is True else "http"
+ metadata_port = app_props["atlas.server.https.port"] if metadata_protocol == "https" else app_props["atlas.server.http.port"]
+ app_props["atlas.rest.address"] = u'%s://%s:%s' % (metadata_protocol, host_name, metadata_port)
+ app_props["atlas.server.ids"] = "id1"
+ app_props["atlas.server.address.id1"] = u"%s:%s" % (host_name, metadata_port)
+ app_props["atlas.server.ha.enabled"] = "false"
+
+ self.assertResourceCalled('File', '/etc/atlas/conf/atlas-log4j.xml',
+ content=InlineTemplate(
+ self.getConfig()['configurations'][
+ 'atlas-log4j']['content']),
+ owner='atlas',
+ group='hadoop',
+ mode=0644,
+ )
+ self.assertResourceCalled('File', '/etc/atlas/conf/atlas-env.sh',
+ content=InlineTemplate(
+ self.getConfig()['configurations'][
+ 'atlas-env']['content']),
+ owner='atlas',
+ group='hadoop',
+ mode=0755,
+ )
+ self.assertResourceCalled('File', '/etc/atlas/conf/solr/solrconfig.xml',
+ content=InlineTemplate(
+ self.getConfig()['configurations'][
+ 'atlas-solrconfig']['content']),
+ owner='atlas',
+ group='hadoop',
+ mode=0644,
+ )
+ # application.properties file
+ self.assertResourceCalled('PropertiesFile',
+ '/etc/atlas/conf/application.properties',
+ properties=app_props,
+ owner=u'atlas',
+ group=u'hadoop',
+ mode=0644,
+ )
+
+ self.assertResourceCalled('TemplateConfig', '/etc/atlas/conf/atlas_jaas.conf',
+ owner = 'atlas',
+ )
+
+ self.assertResourceCalled('Directory', '/var/log/ambari-infra-solr-client',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Directory', '/usr/lib/ambari-infra-solr-client',
+ create_parents = True,
+ recursive_ownership = True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/solrCloudCli.sh',
+ content=StaticFile('/usr/lib/ambari-infra-solr-client/solrCloudCli.sh'),
+ mode=0755,
+ )
+ self.assertResourceCalled('File', '/usr/lib/ambari-infra-solr-client/log4j.properties',
+ content=InlineTemplate(self.getConfig()['configurations'][
+ 'infra-solr-client-log4j']['content']),
+ mode=0644,
+ )
+ self.assertResourceCalled('File', '/var/log/ambari-infra-solr-client/solr-client.log',
+ mode=0664,
+ content=''
+ )
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']),
+ only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5',
+ only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5',
+ not_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ action=['delete'],
+ create_parents=True)
+
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
classname = "MetadataServer",
@@ -177,6 +320,10 @@ class TestMetadataServer(RMFTestCase):
)
self.configureResourcesCalled()
+
+ self.assertResourceCalled('File', '/tmp/atlas_hbase_setup.rb',
+ content=Template("atlas_hbase_setup.rb.j2"))
+
self.assertNoMoreResources()
def test_configure_secure(self):
@@ -188,10 +335,11 @@ class TestMetadataServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.configureResourcesCalled()
- self.assertResourceCalled('TemplateConfig', '/etc/atlas/conf/atlas_jaas.conf',
- owner = 'atlas',
- )
+ self.configureResourcesCalledSecure()
+
+ self.assertResourceCalled('File', '/tmp/atlas_hbase_setup.rb',
+ content=Template("atlas_hbase_setup.rb.j2"))
+
self.assertNoMoreResources()
def test_start_default(self):
@@ -203,6 +351,10 @@ class TestMetadataServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.configureResourcesCalled()
+
+ self.assertResourceCalled('File', '/tmp/atlas_hbase_setup.rb',
+ content=Template("atlas_hbase_setup.rb.j2"))
+
self.assertResourceCalled('Execute', 'source /etc/atlas/conf/atlas-env.sh ; /usr/hdp/current/atlas-server/bin/atlas_start.py',
not_if = 'ls /var/run/atlas/atlas.pid >/dev/null 2>&1 && ps -p `cat /var/run/atlas/atlas.pid` >/dev/null 2>&1',
user = 'atlas',
http://git-wip-us.apache.org/repos/asf/ambari/blob/c17d9801/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index 7fdf3e6..b256bb2 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -176,6 +176,10 @@ class TestAtlasServer(RMFTestCase):
)
self.configureResourcesCalled()
+
+ self.assertResourceCalled('File', '/tmp/atlas_hbase_setup.rb',
+ content=Template("atlas_hbase_setup.rb.j2"))
+
self.assertNoMoreResources()
@patch("resource_management.libraries.functions.security_commons.build_expectations")
[15/55] [abbrv] ambari git commit: AMBARI-18188. KAFKA is spelled as
'KAKFA' in stack_advisor.py because of which the function validate (Anita
Jebaraj via rlevas)
Posted by ol...@apache.org.
AMBARI-18188. KAFKA is spelled as 'KAKFA' in stack_advisor.py because of which the function validate (Anita Jebaraj via rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/712a12b6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/712a12b6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/712a12b6
Branch: refs/heads/branch-dev-logsearch
Commit: 712a12b6b3228d5bd532aa7a594544c8bd594d66
Parents: 1d3f8a4
Author: Anita Jebaraj <aj...@us.ibm.com>
Authored: Thu Aug 18 13:47:26 2016 -0400
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Thu Aug 18 13:48:47 2016 -0400
----------------------------------------------------------------------
.../src/main/resources/stacks/HDP/2.3/services/stack_advisor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/712a12b6/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index a0cce7b..64e8e03 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -749,7 +749,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
"hive-site": self.validateHiveConfigurations},
"HBASE": {"hbase-site": self.validateHBASEConfigurations},
- "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
+ "KAFKA": {"kafka-broker": self.validateKAFKAConfigurations},
"YARN": {"yarn-site": self.validateYARNConfigurations},
"RANGER": {"admin-properties": self.validateRangerAdminConfigurations,
"ranger-env": self.validateRangerConfigurationsEnv}
[20/55] [abbrv] ambari git commit: Revert "AMBARI-18157: Hive service
check is failing after cluster Kerberization (jluniya)"
Posted by ol...@apache.org.
Revert "AMBARI-18157: Hive service check is failing after cluster Kerberization (jluniya)"
This reverts commit 0e5fb8ee8e205ccb9da889e44ba77ffbde3f86dd.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5a943f49
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5a943f49
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5a943f49
Branch: refs/heads/branch-dev-logsearch
Commit: 5a943f49046d3cb30aa3d74a066374aaf29cef05
Parents: 125be04
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 18 21:45:59 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 18 21:45:59 2016 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 91 --------------------
.../stacks/HDP/2.1/services/stack_advisor.py | 20 +----
.../stacks/2.0.6/common/test_stack_advisor.py | 53 ------------
3 files changed, 1 insertion(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a943f49/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7aeda19..7133c27 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1679,97 +1679,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
leafQueueNames.add(leafQueueName)
return leafQueueNames
- def get_service_component_meta(self, service, component, services):
- """
- Function retrieve service component meta information as dict from services.json
- If no service or component found, would be returned empty dict
-
- Return value example:
- "advertise_version" : true,
- "bulk_commands_display_name" : "",
- "bulk_commands_master_component_name" : "",
- "cardinality" : "1+",
- "component_category" : "CLIENT",
- "component_name" : "HBASE_CLIENT",
- "custom_commands" : [ ],
- "decommission_allowed" : false,
- "display_name" : "HBase Client",
- "has_bulk_commands_definition" : false,
- "is_client" : true,
- "is_master" : false,
- "reassign_allowed" : false,
- "recovery_enabled" : false,
- "service_name" : "HBASE",
- "stack_name" : "HDP",
- "stack_version" : "2.5",
- "hostnames" : [ "host1", "host2" ]
-
- :type service str
- :type component str
- :type services dict
- :rtype dict
- """
- __stack_services = "StackServices"
- __stack_service_components = "StackServiceComponents"
-
- if not services:
- return {}
-
- service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
- if len(service_meta) == 0:
- return {}
-
- service_meta = service_meta[0]
- component_meta = [item for item in service_meta["components"] if item[__stack_service_components]["component_name"] == component]
-
- if len(component_meta) == 0:
- return {}
-
- return component_meta[0][__stack_service_components]
-
- def is_secured_cluster(self, services):
- """
- Detects if cluster is secured or not
- :type services dict
- :rtype bool
- """
- return services and "cluster-env" in services["configurations"] and\
- "security_enabled" in services["configurations"]["cluster-env"]["properties"] and\
- services["configurations"]["cluster-env"]["properties"]["security_enabled"].lower() == "true"
-
- def get_services_list(self, services):
- """
- Returns available services as list
-
- :type services dict
- :rtype list
- """
- if not services:
- return []
-
- return [service["StackServices"]["service_name"] for service in services["services"]]
-
- def get_components_list(self, service, services):
- """
- Return list of components for specific service
- :type service str
- :type services dict
- :rtype list
- """
- __stack_services = "StackServices"
- __stack_service_components = "StackServiceComponents"
-
- if not services:
- return []
-
- service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
- if len(service_meta) == 0:
- return []
-
- service_meta = service_meta[0]
- return [item[__stack_service_components]["component_name"] for item in service_meta["components"]]
-
-
def getOldValue(self, services, configType, propertyName):
if services:
if 'changed-configurations' in services.keys():
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a943f49/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index e194b3f..34b4614 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -107,7 +107,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
dbConnection = self.getDBConnectionString(hiveEnvProperties['hive_database']).format(hiveServerHost['Hosts']['host_name'], hiveSiteProperties['ambari.hive.db.schema.name'])
putHiveProperty('javax.jdo.option.ConnectionURL', dbConnection)
- servicesList = self.get_services_list(services)
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if "PIG" in servicesList:
ambari_user = self.getAmbariUser(services)
ambariHostName = socket.getfqdn()
@@ -120,24 +120,6 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
- if self.is_secured_cluster(services):
- appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
-
- def updateCallback(originalValue, newValue):
- if originalValue and not originalValue.isspace():
- hosts = originalValue.split(',')
-
- if newValue:
- hosts.extend(newValue)
-
- result = ','.join(set(hosts))
- return result
- else:
- return newValue
-
- meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
- if "hostnames" in meta:
- appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
putTezProperty = self.putProperty(configurations, "tez-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a943f49/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index a70922f..e28389e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -188,59 +188,6 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
- def test_get_components_list(self):
- servicesInfo = [
- {
- "name": "GANGLIA",
- "components": [
- {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
- {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
- ]
- }
- ]
- expected = sorted(["GANGLIA_MONITOR", "GANGLIA_SERVER"])
-
- services = self.prepareServices(servicesInfo)
- result = sorted(self.stackAdvisor.get_components_list("GANGLIA", services))
-
- self.assertEqual(expected, result)
-
- def test_get_services_list(self):
- servicesInfo = [
- {
- "name": "GANGLIA",
- "components": [
- {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
- {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
- ]
- }
- ]
- expected = ["GANGLIA"]
-
- services = self.prepareServices(servicesInfo)
- result = self.stackAdvisor.get_services_list(services)
-
- self.assertEqual(expected, result)
-
- def test_get_service_component_meta(self):
- servicesInfo = [
- {
- "name": "GANGLIA",
- "components": [
- {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
- {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
- ]
- }
- ]
- expected = ["host1"]
-
- services = self.prepareServices(servicesInfo)
- result = self.stackAdvisor.get_service_component_meta("GANGLIA", "GANGLIA_MONITOR", services)
-
- self.assertEquals(True, "hostnames" in result)
- self.assertEqual(expected, result["hostnames"])
-
-
def test_validationWarnMessagesIfLessThanDefault(self):
servicesInfo = [
{
[34/55] [abbrv] ambari git commit: AMBARI-18219. Ambari should use
oozied.sh for stopping oozie so that optional catalina args can be provided
(Venkat Ranganathan via smohanty)
Posted by ol...@apache.org.
AMBARI-18219. Ambari should use oozied.sh for stopping oozie so that optional catalina args can be provided (Venkat Ranganathan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cfef220d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cfef220d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cfef220d
Branch: refs/heads/branch-dev-logsearch
Commit: cfef220d92ed102cde2ed77fe6d12bd973a189a2
Parents: 4c37044
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Aug 20 12:11:10 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Aug 20 12:11:10 2016 -0700
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie_service.py | 2 +-
.../src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cfef220d/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 74b639b..8371ed1 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -158,7 +158,7 @@ def oozie_service(action = 'start', upgrade_type=None):
raise
elif action == 'stop':
- stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-stop.sh")
+ stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 force")
try:
# stop oozie
http://git-wip-us.apache.org/repos/asf/ambari/blob/cfef220d/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index f38444b..db7eca3 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -576,7 +576,7 @@ class TestOozieServer(RMFTestCase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-stop.sh',
+ self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 force',
environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
user = 'oozie',
@@ -706,7 +706,7 @@ class TestOozieServer(RMFTestCase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-stop.sh',
+ self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 force',
environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
user = 'oozie',
[27/55] [abbrv] ambari git commit: AMBARI-18216. Both NameNodes
reporting version mismatches before Finalizing RU (ncole)
Posted by ol...@apache.org.
AMBARI-18216. Both NameNodes reporting version mismatches before Finalizing RU (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4773c839
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4773c839
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4773c839
Branch: refs/heads/branch-dev-logsearch
Commit: 4773c839965068bb2827b0d308c0bec9599b39c1
Parents: 5f9c3ab
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Aug 19 12:52:01 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Aug 19 15:07:17 2016 -0400
----------------------------------------------------------------------
.../ambari/server/stack/MasterHostResolver.java | 11 ++-
.../ambari/server/state/UpgradeHelper.java | 3 +
.../apache/ambari/server/utils/HTTPUtils.java | 10 ++-
.../ambari/server/state/UpgradeHelperTest.java | 86 +++++++++++++++++++-
4 files changed, 101 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4773c839/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
index e6154d2..f5eb007 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
@@ -23,6 +23,7 @@ import java.net.MalformedURLException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -39,6 +40,7 @@ import org.apache.ambari.server.state.UpgradeState;
import org.apache.ambari.server.utils.HTTPUtils;
import org.apache.ambari.server.utils.HostAndPort;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,7 +146,14 @@ public class MasterHostResolver {
hostsType.master = pair.containsKey(Status.ACTIVE) ? pair.get(Status.ACTIVE) : null;
hostsType.secondary = pair.containsKey(Status.STANDBY) ? pair.get(Status.STANDBY) : null;
} else {
- hostsType.master = componentHosts.iterator().next();
+ // !!! we KNOW we have 2 componentHosts if we're here.
+ Iterator<String> iterator = componentHosts.iterator();
+ hostsType.master = iterator.next();
+ hostsType.secondary = iterator.next();
+
+ LOG.warn("Could not determine the active/standby states from NameNodes {}. " +
+ "Using {} as active and {} as standby.",
+ StringUtils.join(componentHosts, ','), hostsType.master, hostsType.secondary);
}
}
break;
http://git-wip-us.apache.org/repos/asf/ambari/blob/4773c839/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 6cefa79..3a2dc89 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -409,6 +409,9 @@ public class UpgradeHelper {
builder.add(context, hostsType, service.serviceName,
svc.isClientOnlyService(), pc, null);
+ } else {
+ LOG.warn("Could not orchestrate NameNode. Hosts could not be resolved: hosts={}, active={}, standby={}",
+ StringUtils.join(hostsType.hosts, ','), hostsType.master, hostsType.secondary);
}
break;
case NON_ROLLING:
http://git-wip-us.apache.org/repos/asf/ambari/blob/4773c839/ambari-server/src/main/java/org/apache/ambari/server/utils/HTTPUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/HTTPUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/HTTPUtils.java
index cfb7128..fec4c1b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/HTTPUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/HTTPUtils.java
@@ -17,9 +17,6 @@
*/
package org.apache.ambari.server.utils;
-import org.apache.ambari.server.controller.internal.URLStreamProvider;
-import org.apache.ambari.server.proxy.ProxyService;
-
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
@@ -29,6 +26,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
+import org.apache.ambari.server.proxy.ProxyService;
+
/**
* Static Helper methods for HTTP requests.
*/
@@ -46,7 +47,8 @@ public class HTTPUtils {
String url = urlToRead;
try {
- URLStreamProvider urlStreamProvider = new URLStreamProvider(ProxyService.URL_CONNECT_TIMEOUT, ProxyService.URL_READ_TIMEOUT, null, null, null);
+ URLStreamProvider urlStreamProvider = new URLStreamProvider(ProxyService.URL_CONNECT_TIMEOUT,
+ ProxyService.URL_READ_TIMEOUT, ComponentSSLConfiguration.instance());
Map<String, List<String>> headers = new HashMap<String, List<String>>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/4773c839/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 55cb23b..34bac2a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -1598,6 +1598,74 @@ public class UpgradeHelperTest {
assertTrue(ht.hosts.contains("h1"));
}
+ @Test
+ public void testResolverBadJmx() throws Exception {
+ Clusters clusters = injector.getInstance(Clusters.class);
+ ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class);
+
+ String clusterName = "c1";
+ String version = "2.1.1.0-1234";
+
+ StackId stackId = new StackId("HDP-2.1.1");
+ clusters.addCluster(clusterName, stackId);
+ Cluster c = clusters.getCluster(clusterName);
+
+ helper.getOrCreateRepositoryVersion(stackId,
+ c.getDesiredStackVersion().getStackVersion());
+
+ c.createClusterVersion(stackId,
+ c.getDesiredStackVersion().getStackVersion(), "admin",
+ RepositoryVersionState.INSTALLING);
+
+ for (int i = 0; i < 2; i++) {
+ String hostName = "h" + (i+1);
+ clusters.addHost(hostName);
+ Host host = clusters.getHost(hostName);
+
+ Map<String, String> hostAttributes = new HashMap<String, String>();
+ hostAttributes.put("os_family", "redhat");
+ hostAttributes.put("os_release_version", "6");
+
+ host.setHostAttributes(hostAttributes);
+
+ host.persist();
+ clusters.mapHostToCluster(hostName, clusterName);
+ }
+
+ // Add services
+ c.addService(serviceFactory.createNew(c, "HDFS"));
+
+ Service s = c.getService("HDFS");
+ ServiceComponent sc = s.addServiceComponent("NAMENODE");
+ sc.addServiceComponentHost("h1");
+ sc.addServiceComponentHost("h2");
+
+ List<ServiceComponentHost> schs = c.getServiceComponentHosts("HDFS", "NAMENODE");
+ assertEquals(2, schs.size());
+
+ setConfigMocks();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.internal.nameservices")).andReturn("ha").anyTimes();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes();
+
+ // Notice that these names are all caps.
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes();
+ replay(m_configHelper);
+
+ MasterHostResolver mhr = new BadMasterHostResolver(m_configHelper, c, version);
+
+ HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE");
+ assertNotNull(ht.master);
+ assertNotNull(ht.secondary);
+ assertEquals(2, ht.hosts.size());
+
+ // Should be stored in lowercase.
+ assertTrue(ht.hosts.contains("h1"));
+ assertTrue(ht.hosts.contains("h2"));
+ }
+
+
/**
* Tests that advanced {@link Grouping} instances like {@link StopGrouping}
* work with rolling upgrade packs.
@@ -1743,10 +1811,6 @@ public class UpgradeHelperTest {
*/
private class MockMasterHostResolver extends MasterHostResolver {
- public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster) {
- this(configHelper, cluster, null);
- }
-
public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version) {
super(configHelper, cluster, version);
}
@@ -1786,4 +1850,18 @@ public class UpgradeHelperTest {
binder.bind(ConfigHelper.class).toInstance(m_configHelper);
}
}
+
+ private static class BadMasterHostResolver extends MasterHostResolver {
+
+ public BadMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version) {
+ super(configHelper, cluster, version);
+ }
+
+ @Override
+ protected String queryJmxBeanValue(String hostname, int port, String beanName,
+ String attributeName, boolean asQuery, boolean encrypted) {
+ return null;
+ }
+
+ }
}
[22/55] [abbrv] ambari git commit: AMBARI-18157. Hive service check
is failing after cluster Kerberization (dgrinenko via dlysnichenko)
Posted by ol...@apache.org.
AMBARI-18157. Hive service check is failing after cluster Kerberization (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2ad7c802
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2ad7c802
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2ad7c802
Branch: refs/heads/branch-dev-logsearch
Commit: 2ad7c8024403be251235e49898d0ef1203ad6563
Parents: 356feed
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Aug 19 11:33:37 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Aug 19 11:33:37 2016 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 91 ++++++++++++++++++++
.../stacks/HDP/2.1/services/stack_advisor.py | 24 +++++-
.../stacks/2.0.6/common/test_stack_advisor.py | 53 ++++++++++++
.../stacks/2.1/common/test_stack_advisor.py | 83 ++++++++++++++++++
4 files changed, 250 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7133c27..7aeda19 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1679,6 +1679,97 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
leafQueueNames.add(leafQueueName)
return leafQueueNames
+ def get_service_component_meta(self, service, component, services):
+ """
+ Function retrieve service component meta information as dict from services.json
+ If no service or component found, would be returned empty dict
+
+ Return value example:
+ "advertise_version" : true,
+ "bulk_commands_display_name" : "",
+ "bulk_commands_master_component_name" : "",
+ "cardinality" : "1+",
+ "component_category" : "CLIENT",
+ "component_name" : "HBASE_CLIENT",
+ "custom_commands" : [ ],
+ "decommission_allowed" : false,
+ "display_name" : "HBase Client",
+ "has_bulk_commands_definition" : false,
+ "is_client" : true,
+ "is_master" : false,
+ "reassign_allowed" : false,
+ "recovery_enabled" : false,
+ "service_name" : "HBASE",
+ "stack_name" : "HDP",
+ "stack_version" : "2.5",
+ "hostnames" : [ "host1", "host2" ]
+
+ :type service str
+ :type component str
+ :type services dict
+ :rtype dict
+ """
+ __stack_services = "StackServices"
+ __stack_service_components = "StackServiceComponents"
+
+ if not services:
+ return {}
+
+ service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+ if len(service_meta) == 0:
+ return {}
+
+ service_meta = service_meta[0]
+ component_meta = [item for item in service_meta["components"] if item[__stack_service_components]["component_name"] == component]
+
+ if len(component_meta) == 0:
+ return {}
+
+ return component_meta[0][__stack_service_components]
+
+ def is_secured_cluster(self, services):
+ """
+ Detects if cluster is secured or not
+ :type services dict
+ :rtype bool
+ """
+ return services and "cluster-env" in services["configurations"] and\
+ "security_enabled" in services["configurations"]["cluster-env"]["properties"] and\
+ services["configurations"]["cluster-env"]["properties"]["security_enabled"].lower() == "true"
+
+ def get_services_list(self, services):
+ """
+ Returns available services as list
+
+ :type services dict
+ :rtype list
+ """
+ if not services:
+ return []
+
+ return [service["StackServices"]["service_name"] for service in services["services"]]
+
+ def get_components_list(self, service, services):
+ """
+ Return list of components for specific service
+ :type service str
+ :type services dict
+ :rtype list
+ """
+ __stack_services = "StackServices"
+ __stack_service_components = "StackServiceComponents"
+
+ if not services:
+ return []
+
+ service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+ if len(service_meta) == 0:
+ return []
+
+ service_meta = service_meta[0]
+ return [item[__stack_service_components]["component_name"] for item in service_meta["components"]]
+
+
def getOldValue(self, services, configType, propertyName):
if services:
if 'changed-configurations' in services.keys():
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 34b4614..9678dc1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -107,7 +107,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
dbConnection = self.getDBConnectionString(hiveEnvProperties['hive_database']).format(hiveServerHost['Hosts']['host_name'], hiveSiteProperties['ambari.hive.db.schema.name'])
putHiveProperty('javax.jdo.option.ConnectionURL', dbConnection)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ servicesList = self.get_services_list(services)
if "PIG" in servicesList:
ambari_user = self.getAmbariUser(services)
ambariHostName = socket.getfqdn()
@@ -120,6 +120,28 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
+ if self.is_secured_cluster(services):
+ appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
+
+ def updateCallback(originalValue, newValue):
+ """
+ :type originalValue str
+ :type newValue list
+ """
+ if originalValue and not originalValue.isspace():
+ hosts = originalValue.split(',')
+
+ if newValue:
+ hosts.extend(newValue)
+
+ result = ','.join(set(hosts))
+ return result
+ else:
+ return ','.join(set(newValue))
+
+ meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
+ if "hostnames" in meta:
+ appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
putTezProperty = self.putProperty(configurations, "tez-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index e28389e..a70922f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -188,6 +188,59 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
+ def test_get_components_list(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = sorted(["GANGLIA_MONITOR", "GANGLIA_SERVER"])
+
+ services = self.prepareServices(servicesInfo)
+ result = sorted(self.stackAdvisor.get_components_list("GANGLIA", services))
+
+ self.assertEqual(expected, result)
+
+ def test_get_services_list(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = ["GANGLIA"]
+
+ services = self.prepareServices(servicesInfo)
+ result = self.stackAdvisor.get_services_list(services)
+
+ self.assertEqual(expected, result)
+
+ def test_get_service_component_meta(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = ["host1"]
+
+ services = self.prepareServices(servicesInfo)
+ result = self.stackAdvisor.get_service_component_meta("GANGLIA", "GANGLIA_MONITOR", services)
+
+ self.assertEquals(True, "hostnames" in result)
+ self.assertEqual(expected, result["hostnames"])
+
+
def test_validationWarnMessagesIfLessThanDefault(self):
servicesInfo = [
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/2ad7c802/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
index e174461..7835262 100644
--- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
@@ -270,6 +270,89 @@ class TestHDP21StackAdvisor(TestCase):
self.assertEquals(configurations['hive-site']['properties']['javax.jdo.option.ConnectionURL'], "jdbc:mysql://example.com/hive_name")
self.assertEquals(configurations['hive-site']['properties']['javax.jdo.option.ConnectionDriverName'], "com.mysql.jdbc.Driver")
+ def test_recommendHiveConfigurationsSecure(self):
+ services = {
+ "services": [
+ {
+ "StackServices": {
+ "service_name": "HIVE",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "WEBHCAT_SERVER",
+ "service_name": "HIVE",
+ "hostnames": ["example.com"]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations": {}
+ }
+
+ configurations = {
+ "hive-site": {
+ "properties": {
+ "javax.jdo.option.ConnectionDriverName": "",
+ "ambari.hive.db.schema.name": "hive_name",
+ "javax.jdo.option.ConnectionURL": "jdbc:mysql://localhost/hive?createDatabaseIfNotExist=true"
+ }
+ },
+ "hive-env": {
+ "properties": {
+ "hive_database": "New MySQL Database"
+ }
+ },
+ "cluster-env": {
+ "properties": {
+ "security_enabled": "true"
+ }
+ }
+ }
+
+ services['configurations'] = configurations
+ clusterData = {
+ "mapMemory": 3000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ services['changed-configurations'] = []
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "host_name": "example.com"
+ }
+ },
+ {
+ "Hosts": {
+ "host_name": "example.org"
+ }
+ }
+ ]
+ }
+
+ # new mysql
+ self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals("core-site" in configurations, True)
+ self.assertEqual("hadoop.proxyuser.HTTP.hosts" in configurations["core-site"]["properties"], True)
+ self.assertEqual(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] == "example.com", True)
+
+ newhost_list = ["example.com", "example.org"]
+ services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = newhost_list
+ configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] = ""
+
+ self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals("core-site" in configurations, True)
+ self.assertEqual("hadoop.proxyuser.HTTP.hosts" in configurations["core-site"]["properties"], True)
+
+ fetch_list = sorted(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"].split(","))
+ self.assertEqual(sorted(newhost_list), fetch_list)
+
+
+
def test_recommendHiveConfigurations_containersRamIsLess(self):
configurations = {}
clusterData = {
[25/55] [abbrv] ambari git commit: AMBARI-18215 HiveServer2 dies
since ZK unavailable,
Ambari 2.0.2 with Ranger + Kerberos is upgraded to Ambari 2.4.0.0 (dsen)
Posted by ol...@apache.org.
AMBARI-18215 HiveServer2 dies since ZK unavailable, Ambari 2.0.2 with Ranger + Kerberos is upgraded to Ambari 2.4.0.0 (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e7c85c7d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e7c85c7d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e7c85c7d
Branch: refs/heads/branch-dev-logsearch
Commit: e7c85c7dd67ed6633d78f5d427d5095d00a5447d
Parents: 407c452
Author: Dmytro Sen <ds...@apache.org>
Authored: Fri Aug 19 19:47:01 2016 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Fri Aug 19 19:47:01 2016 +0300
----------------------------------------------------------------------
.../src/main/resources/stacks/HDP/2.0.6/role_command_order.json | 2 +-
.../src/main/resources/stacks/HDP/2.2/role_command_order.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e7c85c7d/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json
index a919df9..365aceb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/role_command_order.json
@@ -11,7 +11,7 @@
"HIVE_METASTORE-START": ["MYSQL_SERVER-START", "NAMENODE-START"],
"HIVE_METASTORE-RESTART": ["MYSQL_SERVER-RESTART", "NAMENODE-RESTART"],
"HIVE_SERVER-START": ["NODEMANAGER-START", "MYSQL_SERVER-START"],
- "HIVE_SERVER-RESTART": ["NODEMANAGER-RESTART", "MYSQL_SERVER-RESTART"],
+ "HIVE_SERVER-RESTART": ["NODEMANAGER-RESTART", "MYSQL_SERVER-RESTART", "ZOOKEEPER_SERVER-RESTART"],
"HUE_SERVER-START": ["HIVE_SERVER-START", "HCAT-START", "OOZIE_SERVER-START"],
"FLUME_HANDLER-START": ["OOZIE_SERVER-START"],
"MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/e7c85c7d/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
index efa3b0d..e280550 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
@@ -28,7 +28,7 @@
"RESOURCEMANAGER-START" : ["RANGER_USERSYNC-START"],
"NIMBUS-START": ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START"],
"HBASE_MASTER-START": ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START"],
- "HIVE_SERVER-START": ["NODEMANAGER-START", "MYSQL_SERVER-START", "RANGER_USERSYNC-START"]
+ "HIVE_SERVER-START": ["NODEMANAGER-START", "MYSQL_SERVER-START", "RANGER_USERSYNC-START", "ZOOKEEPER_SERVER-START"]
},
"_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster",
"optional_no_glusterfs": {
[39/55] [abbrv] ambari git commit: AMBARI-18208. Bug Fixing in
HueMIgration View. (Pradarttana Panda via gauravn7)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
index 5d99b49..c9cfc9d 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
@@ -41,7 +41,7 @@ import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceHueDatab
import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.pig.jobqueryset.*;
import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.jobqueryset.*;
-public class PigJobMigrationUtility {
+public class PigJobMigrationUtility {
protected MigrationResourceManager resourceManager = null;
@@ -70,40 +70,28 @@ public class PigJobMigrationUtility {
PigJobMigrationImplementation pigjobimpl = new PigJobMigrationImplementation();// creating the implementation object
- QuerySet huedatabase=null;
+ QuerySet huedatabase = null;
- if(view.getProperties().get("huedrivername").contains("mysql"))
- {
- huedatabase=new MysqlQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("postgresql"))
- {
- huedatabase=new PostgressQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("sqlite"))
- {
- huedatabase=new SqliteQuerySet();
- }
- else if (view.getProperties().get("huedrivername").contains("oracle"))
- {
- huedatabase=new OracleQuerySet();
+ if (view.getProperties().get("huedrivername").contains("mysql")) {
+ huedatabase = new MysqlQuerySet();
+ } else if (view.getProperties().get("huedrivername").contains("postgresql")) {
+ huedatabase = new PostgressQuerySet();
+ } else if (view.getProperties().get("huedrivername").contains("sqlite")) {
+ huedatabase = new SqliteQuerySet();
+ } else if (view.getProperties().get("huedrivername").contains("oracle")) {
+ huedatabase = new OracleQuerySet();
}
- QuerySetAmbariDB ambaridatabase=null;
+ QuerySetAmbariDB ambaridatabase = null;
- if(view.getProperties().get("ambaridrivername").contains("mysql"))
- {
- ambaridatabase=new MysqlQuerySetAmbariDB();
- }
- else if(view.getProperties().get("ambaridrivername").contains("postgresql"))
- {
- ambaridatabase=new PostgressQuerySetAmbariDB();
+ if (view.getProperties().get("ambaridrivername").contains("mysql")) {
+ ambaridatabase = new MysqlQuerySetAmbariDB();
+ } else if (view.getProperties().get("ambaridrivername").contains("postgresql")) {
+ ambaridatabase = new PostgressQuerySetAmbariDB();
+ } else if (view.getProperties().get("ambaridrivername").contains("oracle")) {
+ ambaridatabase = new OracleQuerySetAmbariDB();
}
- else if (view.getProperties().get("ambaridrivername").contains("oracle"))
- {
- ambaridatabase= new OracleQuerySetAmbariDB();
- }
- int maxCountforPigScript = 0,i=0;
+ int maxCountforPigScript = 0, i = 0;
String time = null, timeIndorder = null;
Long epochtime = null;
@@ -114,11 +102,10 @@ public class PigJobMigrationUtility {
connectionHuedb = DataSourceHueDatabase.getInstance(view.getProperties().get("huedrivername"), view.getProperties().get("huejdbcurl"), view.getProperties().get("huedbusername"), view.getProperties().get("huedbpassword")).getConnection();//connecting to hue database
- pigJobDbPojo = pigjobimpl.fetchFromHueDB(username, startDate, endDate, connectionHuedb,huedatabase);// fetching the PigJobs details from hue
+ pigJobDbPojo = pigjobimpl.fetchFromHueDB(username, startDate, endDate, connectionHuedb, huedatabase);// fetching the PigJobs details from hue
- for(int j=0;j<pigJobDbPojo.size();j++)
- {
- logger.info("the query fetched from hue="+pigJobDbPojo.get(i).getScript());
+ for (int j = 0; j < pigJobDbPojo.size(); j++) {
+ logger.info("the query fetched from hue=" + pigJobDbPojo.get(i).getScript());
}
@@ -137,54 +124,52 @@ public class PigJobMigrationUtility {
connectionAmbaridb = DataSourceAmbariDatabase.getInstance(view.getProperties().get("ambaridrivername"), view.getProperties().get("ambarijdbcurl"), view.getProperties().get("ambaridbusername"), view.getProperties().get("ambaridbpassword")).getConnection();
connectionAmbaridb.setAutoCommit(false);
+ int fetchPigTablenameInstance = pigjobimpl.fetchInstanceTablename(connectionAmbaridb, instance, ambaridatabase);
+ int sequence = pigjobimpl.fetchSequenceno(connectionAmbaridb, fetchPigTablenameInstance, ambaridatabase);
+
for (i = 0; i < pigJobDbPojo.size(); i++) {
float calc = ((float) (i + 1)) / pigJobDbPojo.size() * 100;
int progressPercentage = Math.round(calc);
migrationresult.setIsNoQuerySelected("no");
migrationresult.setProgressPercentage(progressPercentage);
- migrationresult.setNumberOfQueryTransfered(i+1);
+ migrationresult.setNumberOfQueryTransfered(i + 1);
migrationresult.setTotalNoQuery(pigJobDbPojo.size());
getResourceManager(view).update(migrationresult, jobid);
-
-
-
logger.info("Loop No." + (i + 1));
logger.info("________________");
logger.info("the title of script " + pigJobDbPojo.get(i).getTitle());
- int fetchPigTablenameInstance = pigjobimpl.fetchInstanceTablename(connectionAmbaridb, instance,ambaridatabase);
-
- maxCountforPigScript = (pigjobimpl.fetchMaxIdforPigJob(connectionAmbaridb, fetchPigTablenameInstance,ambaridatabase) + 1);
+ maxCountforPigScript = i + sequence + 1;
time = pigjobimpl.getTime();
timeIndorder = pigjobimpl.getTimeInorder();
epochtime = pigjobimpl.getEpochTime();
- pigJobDirName = "/user/admin/pig/jobs/" + pigJobDbPojo.get(i).getTitle() + "_" + time + "/";
+ pigJobDirName = "/user/"+username+"/pig/jobs/" + pigJobDbPojo.get(i).getTitle() + "_" + time + "/";
- pigjobimpl.insertRowPigJob(pigJobDirName, maxCountforPigScript, time, timeIndorder, epochtime, pigJobDbPojo.get(i).getTitle(), connectionAmbaridb, fetchPigTablenameInstance, pigJobDbPojo.get(i).getStatus(), instance, i,ambaridatabase);
+ pigjobimpl.insertRowPigJob(pigJobDirName, maxCountforPigScript, time, timeIndorder, epochtime, pigJobDbPojo.get(i).getTitle(), connectionAmbaridb, fetchPigTablenameInstance, pigJobDbPojo.get(i).getStatus(), instance, i, ambaridatabase, username);
if (view.getProperties().get("KerberoseEnabled").equals("y")) {
-
- pigjobimpl.createDirPigJobSecured(pigJobDirName, view.getProperties().get("namenode_URI_Ambari"));
- pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/script.pig", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/stderr", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/stdout", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
+ pigjobimpl.createDirPigJobSecured(pigJobDirName, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));
+ pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/script.pig", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"),username,view.getProperties().get("PrincipalUserName"));
+ pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/stderr", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"),username,view.getProperties().get("PrincipalUserName"));
+ pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/stdout", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"),username,view.getProperties().get("PrincipalUserName"));
} else {
- pigjobimpl.createDirPigJob(pigJobDirName, view.getProperties().get("namenode_URI_Ambari"));
- pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/script.pig", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/stderr", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/stdout", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
+ pigjobimpl.createDirPigJob(pigJobDirName, view.getProperties().get("namenode_URI_Ambari"),username);
+ pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/script.pig", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"),username);
+ pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/stderr", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"),username);
+ pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/stdout", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"),username);
}
logger.info(pigJobDbPojo.get(i).getTitle() + "has been migrated to Ambari");
}
+ pigjobimpl.updateSequenceno(connectionAmbaridb, maxCountforPigScript, fetchPigTablenameInstance, ambaridatabase);
connectionAmbaridb.commit();
}
@@ -194,16 +179,16 @@ public class PigJobMigrationUtility {
connectionAmbaridb.rollback();
logger.info("roll back done");
} catch (SQLException e1) {
- logger.error("roll back exception:",e1);
+ logger.error("roll back exception:", e1);
}
} catch (ClassNotFoundException e2) {
- logger.error("class not found exception:",e2);
+ logger.error("class not found exception:", e2);
} catch (ParseException e) {
- logger.error("ParseException: " ,e);
+ logger.error("ParseException: ", e);
} catch (URISyntaxException e) {
- logger.error("URISyntaxException" ,e);
+ logger.error("URISyntaxException", e);
} catch (PropertyVetoException e) {
- logger.error("PropertyVetoException" ,e);
+ logger.error("PropertyVetoException", e);
} finally {
if (null != connectionAmbaridb)
try {
@@ -217,13 +202,6 @@ public class PigJobMigrationUtility {
logger.info("pig Job Migration End");
logger.info("------------------------------");
- //session.setAttribute(ProgressBarStatus.TASK_PROGRESS_VARIABLE, 0);
-
-// CheckProgresStatus.setProgressPercentage(0);
-// CheckProgresStatus.setNoOfQueryCompleted(0);
-// CheckProgresStatus.setTotalNoOfQuery(0);
-// CheckProgresStatus.setNoOfQueryLeft(0);
-
long stopTime = System.currentTimeMillis();
long elapsedTime = stopTime - startTime;
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
index c5f073c..eaf5d38 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
@@ -43,8 +43,6 @@ public class PigSavedScriptStartJob extends Thread{
this.view=view;
}
-
-
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
index c8aa1c0..b259cfb 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
@@ -39,6 +39,7 @@ import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
import java.io.*;
+import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.sql.*;
import java.text.ParseException;
@@ -127,6 +128,29 @@ public class PigScriptMigrationImplementation {
}
+ public int fetchSequenceno(Connection c, int id, QuerySetAmbariDB ambaridatabase) throws SQLException {
+
+ String ds_id = new String();
+ Statement stmt = null;
+ PreparedStatement prSt = null;
+ int sequencevalue=0;
+
+
+ ResultSet rs = null;
+
+
+ prSt = ambaridatabase.getSequenceNoFromAmbariSequence(c, id);
+
+ logger.info("sql statement to fetch is from ambari instance:= = " + prSt);
+
+ rs = prSt.executeQuery();
+
+ while (rs.next()) {
+ sequencevalue = rs.getInt("sequence_value");
+ }
+ return sequencevalue;
+ }
+
public int fetchInstanceTablenamePigScript(Connection c, String instance, QuerySetAmbariDB ambaridatabase) throws SQLException {
String ds_id = new String();
@@ -175,7 +199,16 @@ public class PigScriptMigrationImplementation {
return num;
}
- public void insertRowForPigScript(String dirname, int maxcountforpigjob, int maxcount, String time, String time2, long epochtime, String title, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+ public void updateSequenceno(Connection c, int seqNo, int id, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+
+ PreparedStatement prSt;
+ prSt = ambaridatabase.updateSequenceNoInAmbariSequence(c, seqNo, id);
+ logger.info("The actual insert statement is " + prSt);
+ prSt.executeUpdate();
+ logger.info("adding revert sql hive history");
+ }
+
+ public void insertRowForPigScript(String dirname, int maxcountforpigjob, int maxcount, String time, String time2, long epochtime, String title, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase,String username) throws SQLException, IOException {
String maxcount1 = Integer.toString(maxcount);
String epochtime1 = Long.toString(epochtime);
@@ -183,7 +216,7 @@ public class PigScriptMigrationImplementation {
PreparedStatement prSt = null;
- prSt = ambaridatabase.insertToPigScript(c, id, maxcount1, dirname, title);
+ prSt = ambaridatabase.insertToPigScript(c, id, maxcount1, dirname, title,username);
prSt.executeUpdate();
@@ -383,7 +416,77 @@ public class PigScriptMigrationImplementation {
}
- public void putFileinHdfs(final String source, final String dest, final String namenodeuri)
+ public void createDirPigScriptSecured(final String dir, final String namenodeuri,final String username,final String principalName)
+ throws IOException, URISyntaxException {
+ try {
+ final Configuration conf = new Configuration();
+
+ conf.set("fs.hdfs.impl",
+ org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()
+ );
+ conf.set("fs.file.impl",
+ org.apache.hadoop.fs.LocalFileSystem.class.getName()
+ );
+ conf.set("fs.defaultFS", namenodeuri);
+ conf.set("hadoop.security.authentication", "Kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
+ ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+
+ public Boolean run() throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+ Path src = new Path(dir);
+ Boolean b = fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
+ return b;
+ }
+ });
+ } catch (Exception e) {
+ logger.error("Exception in Webhdfs", e);
+ }
+ }
+
+ public void createDirPigScript(final String dir, final String namenodeuri,final String username)
+ throws IOException, URISyntaxException {
+
+ try {
+ final Configuration conf = new Configuration();
+
+ conf.set("fs.hdfs.impl",
+ org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()
+ );
+ conf.set("fs.file.impl",
+ org.apache.hadoop.fs.LocalFileSystem.class.getName()
+ );
+ conf.set("fs.defaultFS", namenodeuri);
+ conf.set("hadoop.job.ugi", "hdfs");
+ conf.set("hadoop.security.authentication", "Kerberos");
+
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
+
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+
+ public Void run() throws Exception {
+
+ FileSystem fs = FileSystem.get(conf);
+ Path src = new Path(dir);
+ fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ logger.error("Webhdfs: ", e);
+ }
+ }
+
+
+
+
+ public void putFileinHdfs(final String source, final String dest, final String namenodeuri,final String username)
throws IOException {
try {
@@ -430,6 +533,7 @@ public class PigScriptMigrationImplementation {
}
in.close();
out.close();
+ fileSystem.setOwner(path,username,"hadoop");
fileSystem.close();
return null;
}
@@ -440,10 +544,11 @@ public class PigScriptMigrationImplementation {
}
- public void putFileinHdfsSecured(final String source, final String dest, final String namenodeuri)
+ public void putFileinHdfsSecured(final String source, final String dest, final String namenodeuri,final String username,final String principalName)
throws IOException {
try {
+
final Configuration conf = new Configuration();
conf.set("fs.hdfs.impl",
@@ -453,12 +558,11 @@ public class PigScriptMigrationImplementation {
org.apache.hadoop.fs.LocalFileSystem.class.getName()
);
conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
conf.set("hadoop.security.authentication", "Kerberos");
-
UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
-
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
ugi.doAs(new PrivilegedExceptionAction<Void>() {
public Void run() throws Exception {
@@ -478,7 +582,7 @@ public class PigScriptMigrationImplementation {
if (fileSystem.exists(path)) {
}
- // Path pathsource = new Path(source);
+
FSDataOutputStream out = fileSystem.create(path);
InputStream in = new BufferedInputStream(
@@ -491,12 +595,13 @@ public class PigScriptMigrationImplementation {
}
in.close();
out.close();
+ fileSystem.setOwner(path,username,"hadoop");
fileSystem.close();
return null;
}
});
} catch (Exception e) {
- logger.error("Webhdfs Exception: ", e);
+ logger.error("Webhdfs exception", e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
index 44e27c1..00e1c6c 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
@@ -34,6 +34,7 @@ import org.apache.log4j.Logger;
import java.beans.PropertyVetoException;
import java.io.IOException;
+import java.net.URISyntaxException;
import java.sql.Connection;
import java.sql.SQLException;
import java.text.ParseException;
@@ -98,11 +99,11 @@ public class PigScriptMigrationUtility {
ambaridatabase = new OracleQuerySetAmbariDB();
}
- int maxcountforsavequery = 0, maxcountforpigsavedscript;
+ int maxcountforsavequery = 0, maxcountforpigsavedscript = 0;
String time = null, timetobeInorder = null;
Long epochTime = null;
String dirNameForPigScript, completeDirandFilePath, pigscriptFilename = "";
- int pigInstanceTableName;
+ int pigInstanceTableName, sequence;
ArrayList<PigModel> dbpojoPigSavedscript = new ArrayList<PigModel>();
@@ -132,10 +133,15 @@ public class PigScriptMigrationUtility {
connectionAmbaridb = DataSourceAmbariDatabase.getInstance(view.getProperties().get("ambaridrivername"), view.getProperties().get("ambarijdbcurl"), view.getProperties().get("ambaridbusername"), view.getProperties().get("ambaridbpassword")).getConnection();// connecting to ambari db
connectionAmbaridb.setAutoCommit(false);
+
logger.info("loop will continue for " + dbpojoPigSavedscript.size() + "times");
//for each pig script found in Hue Database
+ pigInstanceTableName = pigsavedscriptmigration.fetchInstanceTablenamePigScript(connectionAmbaridb, instance, ambaridatabase);// finding the table name in ambari from the given instance
+
+ sequence = pigsavedscriptmigration.fetchSequenceno(connectionAmbaridb, pigInstanceTableName, ambaridatabase);
+
for (i = 0; i < dbpojoPigSavedscript.size(); i++) {
@@ -151,17 +157,15 @@ public class PigScriptMigrationUtility {
logger.info("________________");
logger.info("the title of script: " + dbpojoPigSavedscript.get(i).getTitle());
- pigInstanceTableName = pigsavedscriptmigration.fetchInstanceTablenamePigScript(connectionAmbaridb, instance, ambaridatabase);// finding the table name in ambari from the given instance
-
- maxcountforpigsavedscript = (pigsavedscriptmigration.fetchmaxIdforPigSavedScript(connectionAmbaridb, pigInstanceTableName, ambaridatabase) + 1);// maximum count of the primary key of pig Script table
-
time = pigsavedscriptmigration.getTime();
timetobeInorder = pigsavedscriptmigration.getTimeInorder();
epochTime = pigsavedscriptmigration.getEpochTime();
- dirNameForPigScript = "/user/admin/pig/scripts/";
+ maxcountforpigsavedscript = i + sequence + 1;
+
+ dirNameForPigScript = "/user/"+username+"/pig/scripts/";
pigscriptFilename = dbpojoPigSavedscript.get(i).getTitle() + "-" + time + ".pig";
@@ -169,12 +173,16 @@ public class PigScriptMigrationUtility {
pigsavedscriptmigration.writetPigScripttoLocalFile(dbpojoPigSavedscript.get(i).getScript(), dbpojoPigSavedscript.get(i).getTitle(), dbpojoPigSavedscript.get(i).getDt(), ConfigurationCheckImplementation.getHomeDir(), pigscriptFilename);
- pigsavedscriptmigration.insertRowForPigScript(completeDirandFilePath, maxcountforsavequery, maxcountforpigsavedscript, time, timetobeInorder, epochTime, dbpojoPigSavedscript.get(i).getTitle(), connectionAmbaridb, pigInstanceTableName, instance, i, ambaridatabase);
+ pigsavedscriptmigration.insertRowForPigScript(completeDirandFilePath, maxcountforsavequery, maxcountforpigsavedscript, time, timetobeInorder, epochTime, dbpojoPigSavedscript.get(i).getTitle(), connectionAmbaridb, pigInstanceTableName, instance, i, ambaridatabase, username);
if (view.getProperties().get("KerberoseEnabled").equals("y")) {
- pigsavedscriptmigration.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + pigscriptFilename, dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"));
+
+ pigsavedscriptmigration.createDirPigScriptSecured(dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));
+ pigsavedscriptmigration.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + pigscriptFilename, dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));
} else {
- pigsavedscriptmigration.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + pigscriptFilename, dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"));
+
+ pigsavedscriptmigration.createDirPigScript(dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"),username);
+ pigsavedscriptmigration.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + pigscriptFilename, dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"),username);
}
logger.info(dbpojoPigSavedscript.get(i).getTitle() + "Migrated to Ambari");
@@ -182,8 +190,10 @@ public class PigScriptMigrationUtility {
pigsavedscriptmigration.deletePigScriptLocalFile(ConfigurationCheckImplementation.getHomeDir(), pigscriptFilename);
}
+ pigsavedscriptmigration.updateSequenceno(connectionAmbaridb, maxcountforpigsavedscript, pigInstanceTableName, ambaridatabase);
connectionAmbaridb.commit();
+
}
@@ -201,6 +211,8 @@ public class PigScriptMigrationUtility {
logger.error("ParseException: ", e);
} catch (PropertyVetoException e) {
logger.error("PropertyVetoException: ", e);
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
} finally {
if (null != connectionAmbaridb)
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/resources/scripts/models/MigrationModel.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/resources/scripts/models/MigrationModel.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/resources/scripts/models/MigrationModel.java
index f765e15..b341474 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/resources/scripts/models/MigrationModel.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/resources/scripts/models/MigrationModel.java
@@ -34,25 +34,25 @@ public class MigrationModel implements Serializable,PersonalResource{
private String intanceName="";
private String userNameofhue="";
private int totalNoQuery;
- private String ProgressPercentage="";
+ private String progressPercentage="";
private String owner = "";
- private Boolean IfSuccess;
- private String TimeTakentotransfer="";
+ private Boolean ifSuccess;
+ private String timeTakentotransfer="";
public String getTimeTakentotransfer() {
- return TimeTakentotransfer;
+ return timeTakentotransfer;
}
public void setTimeTakentotransfer(String timeTakentotransfer) {
- TimeTakentotransfer = timeTakentotransfer;
+ timeTakentotransfer = timeTakentotransfer;
}
public Boolean getIfSuccess() {
- return IfSuccess;
+ return ifSuccess;
}
public void setIfSuccess(Boolean ifSuccess) {
- IfSuccess = ifSuccess;
+ ifSuccess = ifSuccess;
}
public MigrationModel(Map<String, Object> stringObjectMap) throws InvocationTargetException, IllegalAccessException {
@@ -113,11 +113,11 @@ public class MigrationModel implements Serializable,PersonalResource{
}
public String getProgressPercentage() {
- return ProgressPercentage;
+ return progressPercentage;
}
public void setProgressPercentage(String progressPercentage) {
- ProgressPercentage = progressPercentage;
+ progressPercentage = progressPercentage;
}
public String getOwner() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/checkprogress.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/checkprogress.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/checkprogress.js
index 1f4d2c6..e6e466f 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/checkprogress.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/checkprogress.js
@@ -27,4 +27,4 @@ export default Model.extend({
totalTimeTaken: DS.attr('string'),
jobtype: DS.attr('string'),
isNoQuerySelected: DS.attr('string')
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/returnjobid.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/returnjobid.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/returnjobid.js
index 5d55773..84c30c7 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/returnjobid.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/returnjobid.js
@@ -20,4 +20,4 @@ import DS from 'ember-data';
export default Model.extend({
idforJob: DS.attr('string')
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/startmigration.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/startmigration.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/startmigration.js
index 5913726..f2f57b4 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/startmigration.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/models/startmigration.js
@@ -20,4 +20,4 @@ import DS from 'ember-data';
export default Model.extend({
progressPercentage: DS.attr('string')
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-history.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-history.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-history.js
index a47b93f..158a154 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-history.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-history.js
@@ -17,6 +17,8 @@
*/
import Ember from 'ember';
+
+
export default Ember.Route.extend({
model: function() {
@@ -31,6 +33,11 @@ export default Ember.Route.extend({
actions: {
submitResult: function() {
+
+ if(this.controller.get('usernamehue')===undefined || this.controller.get('instancename') ===undefined){
+ alert("Mandatory fields can not left blank");
+ }
+ else{
this.controller.set('jobstatus', null);
this.controller.set('progressBar', null);
this.controller.set('completionStatus', null);
@@ -61,6 +68,7 @@ export default Ember.Route.extend({
repeat.progresscheck(jobid);
});
});
+ }
}
},
progresscheck: function(jobid) {
@@ -80,6 +88,7 @@ export default Ember.Route.extend({
var userNameofhue = progress.get('userNameofhue');
var totalTimeTaken = progress.get('totalTimeTaken');
var isNoQuerySelected = progress.get('isNoQuerySelected');
+ console.log("the progress percentage is="+progressPercentage);
if (progressPercentage !== '100' && isNoQuerySelected === 'no') {
control.set('progressBar', progressPercentage);
@@ -103,4 +112,4 @@ export default Ember.Route.extend({
});
}, 500);
}
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-saved-query.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-saved-query.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-saved-query.js
index d50a402..2006e8d 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-saved-query.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/hive-saved-query.js
@@ -31,6 +31,11 @@ export default Ember.Route.extend({
actions: {
submitResult: function() {
+
+ if(this.controller.get('usernamehue')===undefined || this.controller.get('instancename') ===undefined){
+ alert("Mandatory fields can not left blank");
+ }
+ else{
this.controller.set('jobstatus', null);
this.controller.set('progressBar', null);
this.controller.set('completionStatus', null);
@@ -61,6 +66,7 @@ export default Ember.Route.extend({
});
});
}
+ }
},
progresscheck: function(jobid) {
var repeat = this;
@@ -99,4 +105,4 @@ export default Ember.Route.extend({
});
}, 500);
}
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-job.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-job.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-job.js
index debb1eb..d2dfd39 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-job.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-job.js
@@ -31,6 +31,11 @@ export default Ember.Route.extend({
actions: {
submitResult: function() {
+
+ if(this.controller.get('usernamehue')===undefined || this.controller.get('instancename') ===undefined){
+ alert("Mandatory fields can not left blank");
+ }
+ else{
this.controller.set('jobstatus', null);
this.controller.set('progressBar', null);
this.controller.set('completionStatus', null);
@@ -61,6 +66,7 @@ export default Ember.Route.extend({
});
});
}
+ }
},
progresscheck: function(jobid) {
var repeat = this;
@@ -99,4 +105,4 @@ export default Ember.Route.extend({
});
}, 500);
}
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-script.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-script.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-script.js
index ab0b919..c121f63 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-script.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/pig-script.js
@@ -31,6 +31,11 @@ export default Ember.Route.extend({
actions: {
submitResult: function() {
+ if(this.controller.get('usernamehue')===undefined || this.controller.get('instancename') ===undefined){
+ alert("Mandatory fields can not left blank");
+ }
+ else
+ {
this.controller.set('jobstatus', null);
this.controller.set('progressBar', null);
this.controller.set('completionStatus', null);
@@ -61,6 +66,7 @@ export default Ember.Route.extend({
});
});
}
+ }
},
progresscheck: function(jobid) {
var repeat = this;
@@ -100,4 +106,4 @@ export default Ember.Route.extend({
});
}, 500);
}
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/revert-change.js
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/revert-change.js b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/revert-change.js
index b34b5ed..06c7542 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/revert-change.js
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/app/routes/home-page/revert-change.js
@@ -29,6 +29,11 @@ export default Ember.Route.extend({
actions: {
submitResult: function() {
+
+ if(this.controller.get('instancename') ===undefined){
+ alert("Mandatory fields can not left blank");
+ }
+ else{
this.controller.set('jobstatus', null);
this.controller.set('progressBar', null);
this.controller.set('completionStatus', null);
@@ -46,11 +51,12 @@ export default Ember.Route.extend({
revertdate: control.get('revertdate'),
jobid: jobid
});
- hivehistoryqueryjobstart.then(function() {
+ hivehistoryqueryjobstart.then(function() {
control.set('jobstatus', "0");
repeat.progresscheck(jobid);
});
});
+ }
}
},
@@ -94,4 +100,4 @@ export default Ember.Route.extend({
}, 500);
}
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/view.xml b/contrib/views/hueambarimigration/src/main/resources/view.xml
index dcfa943..3abbd72 100644
--- a/contrib/views/hueambarimigration/src/main/resources/view.xml
+++ b/contrib/views/hueambarimigration/src/main/resources/view.xml
@@ -122,10 +122,17 @@ limitations under the License. Kerberos, LDAP, Custom. Binary/Htt
<parameter>
<name>KerberoseEnabled</name>
<description>Kerberose enable</description>
- <label>Kerberose enabled on Ambari cluster?(y/n)</label>
+ <label>Kerberos enabled on Ambari cluster?(y/n)</label>
<placeholder>y/n</placeholder>
</parameter>
+ <parameter>
+ <name>PrincipalUserName</name>
+ <description>Kerberos enable</description>
+ <label>principal name (if kerberos is enabled)</label>
+ <placeholder>Please enter the principal name is kerberos is enabled</placeholder>
+ </parameter>
+
<resource>
<name>ambaridatabases</name>
[41/55] [abbrv] ambari git commit: AMBARI-18223. Atlas hook impl jars
should be copied instead of the top level atlas hive hook jars for oozie hive
action (Venkat Ranganathan via alejandro)
Posted by ol...@apache.org.
AMBARI-18223. Atlas hook impl jars should be copied instead of the top level atlas hive hook jars for oozie hive action (Venkat Ranganathan via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e74ecc9a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e74ecc9a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e74ecc9a
Branch: refs/heads/branch-dev-logsearch
Commit: e74ecc9a5a54eb1b3ff2604d2748ef3111856892
Parents: 7a34fe1
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Sun Aug 21 15:21:25 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Sun Aug 21 15:21:25 2016 -0700
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e74ecc9a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 620de92..0736ae9 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -379,8 +379,10 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
"contain directory {atlas_hive_hook_dir}"))
return
- num_files = len([name for name in os.listdir(atlas_hive_hook_dir) if os.path.exists(os.path.join(atlas_hive_hook_dir, name))])
- Logger.info("Found %d files/directories inside Atlas Hive hook directory %s"% (num_files, atlas_hive_hook_dir))
+ atlas_hive_hook_impl_dir = os.path.join(atlas_hive_hook_dir, "atlas-hive-plugin-impl")
+
+ num_files = len([name for name in os.listdir(atlas_hive_hook_impl_dir) if os.path.exists(os.path.join(atlas_hive_hook_impl_dir, name))])
+ Logger.info("Found %d files/directories inside Atlas Hive hook impl directory %s"% (num_files, atlas_hive_hook_impl_dir))
# This can return over 100 files, so take the first 5 lines after "Available ShareLib"
# Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL
@@ -395,12 +397,12 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
raise Fail("Could not parse Hive sharelib from output.")
Logger.info("Parsed Hive sharelib = %s and will attempt to copy/replace %d files to it from %s" %
- (hive_sharelib_dir, num_files, atlas_hive_hook_dir))
+ (hive_sharelib_dir, num_files, atlas_hive_hook_impl_dir))
params.HdfsResource(hive_sharelib_dir,
type="directory",
action="create_on_execute",
- source=atlas_hive_hook_dir,
+ source=atlas_hive_hook_impl_dir,
user=params.hdfs_user,
owner=params.oozie_user,
group=params.hdfs_user,
[35/55] [abbrv] ambari git commit: AMBARI-18221. Oozie server start
fails while enabling wire encryption with Atlas (smohanty)
Posted by ol...@apache.org.
AMBARI-18221. Oozie server start fails while enabling wire encryption with Atlas (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fe768ad8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fe768ad8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fe768ad8
Branch: refs/heads/branch-dev-logsearch
Commit: fe768ad8767fcd82ce13b8d762f987efa8557df2
Parents: cfef220
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Aug 20 18:10:36 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Aug 20 18:11:17 2016 -0700
----------------------------------------------------------------------
.../common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe768ad8/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index c408f8d..620de92 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -383,7 +383,8 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
Logger.info("Found %d files/directories inside Atlas Hive hook directory %s"% (num_files, atlas_hive_hook_dir))
# This can return over 100 files, so take the first 5 lines after "Available ShareLib"
- command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -shareliblist hive | grep "\[Available ShareLib\]" -A 5')
+ # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL
+ command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5')
try:
code, out = call(command, user=params.oozie_user, tries=10, try_sleep=5, logoutput=True)
@@ -424,8 +425,8 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
params.HdfsResource(None, action="execute")
# Update the sharelib after making any changes
- # Since calling oozie-env.sh, don't have to specify -oozie http(s):localhost:{oozie_server_admin_port}/oozie
- command = format("source {conf_dir}/oozie-env.sh ; oozie admin -sharelibupdate")
+ # Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL
+ command = format("source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -sharelibupdate")
code, out = call(command, user=params.oozie_user, tries=5, try_sleep=5, logoutput=True)
if code == 0 and out is not None:
Logger.info("Successfully updated the Oozie ShareLib")
[26/55] [abbrv] ambari git commit: AMBARI-17591 : Collect JVM Heap,
GC and thread pool metrics from Ambari Server and push to AMS.
(Li-Wei Tseng via avijayan)
Posted by ol...@apache.org.
AMBARI-17591 : Collect JVM Heap, GC and thread pool metrics from Ambari Server and push to AMS. (Li-Wei Tseng via avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5f9c3ab1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5f9c3ab1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5f9c3ab1
Branch: refs/heads/branch-dev-logsearch
Commit: 5f9c3ab1a3fbdb775f27adfac5d4cd43443018cd
Parents: e7c85c7
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Fri Aug 19 09:49:22 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Fri Aug 19 09:49:32 2016 -0700
----------------------------------------------------------------------
ambari-server/conf/unix/metrics.properties | 31 +
ambari-server/pom.xml | 10 +
.../ambari/server/controller/AmbariServer.java | 13 +
.../server/controller/ControllerModule.java | 4 +
.../server/metrics/system/AmbariMetricSink.java | 34 +
.../server/metrics/system/MetricsService.java | 26 +
.../server/metrics/system/MetricsSource.java | 25 +
.../system/impl/AbstractMetricsSource.java | 41 ++
.../system/impl/AmbariMetricSinkImpl.java | 161 +++++
.../metrics/system/impl/Configuration.java | 83 +++
.../metrics/system/impl/JvmMetricsSource.java | 75 ++
.../metrics/system/impl/MetricsServiceImpl.java | 168 +++++
.../default/grafana-ambari-server.json | 691 +++++++++++++++++++
13 files changed, 1362 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/conf/unix/metrics.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/metrics.properties b/ambari-server/conf/unix/metrics.properties
new file mode 100644
index 0000000..5f01e39
--- /dev/null
+++ b/ambari-server/conf/unix/metrics.properties
@@ -0,0 +1,31 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Metrics sources info
+metrics.sources=jvm
+
+# Source interval determines how often the metric is sent to sink. Its unit is in seconds
+source.jvm.interval=5
+source.jvm.class=org.apache.ambari.server.metrics.system.impl.JvmMetricsSource
+
+#source.database.interval=10
+#source.database.class=org.apache.ambari.server.metrics.system.impl.DbMetricSource
+
+# Sink frequency determines how often the sink publish the metrics from buffer to AMS.
+sink.frequency=10
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 0d91458..45d3ce4 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1382,6 +1382,16 @@
<artifactId>jul-to-slf4j</artifactId>
<version>1.7.2</version>
</dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>3.1.0</version>
+ </dependency>
</dependencies>
<pluginRepositories>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 097f01c..d20b1d7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -75,6 +75,7 @@ import org.apache.ambari.server.controller.internal.UserPrivilegeResourceProvide
import org.apache.ambari.server.controller.internal.ViewPermissionResourceProvider;
import org.apache.ambari.server.controller.metrics.ThreadPoolEnabledPropertyProvider;
import org.apache.ambari.server.controller.utilities.KerberosChecker;
+import org.apache.ambari.server.metrics.system.MetricsService;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.PersistenceType;
import org.apache.ambari.server.orm.dao.BlueprintDAO;
@@ -158,6 +159,10 @@ import com.google.inject.name.Named;
import com.google.inject.persist.Transactional;
import com.sun.jersey.spi.container.servlet.ServletContainer;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+
@Singleton
public class AmbariServer {
public static final String VIEWS_URL_PATTERN = "/api/v1/views/*";
@@ -616,6 +621,14 @@ public class AmbariServer {
*/
server.start();
+ // TODO, start every other tread.
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ MetricsService metricsService = injector.getInstance(
+ MetricsService.class);
+ metricsService.init();
+ executor.submit(metricsService);
+ LOG.info("********* Started Ambari Metrics **********");
+
serverForAgent.start();
LOG.info("********* Started Server **********");
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
index 256f0d1..bd34c25 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
@@ -77,6 +77,8 @@ import org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetric
import org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricCacheProvider;
import org.apache.ambari.server.controller.spi.ResourceProvider;
import org.apache.ambari.server.controller.utilities.KerberosChecker;
+import org.apache.ambari.server.metrics.system.MetricsService;
+import org.apache.ambari.server.metrics.system.impl.MetricsServiceImpl;
import org.apache.ambari.server.notifications.DispatchFactory;
import org.apache.ambari.server.notifications.NotificationDispatcher;
import org.apache.ambari.server.notifications.dispatchers.SNMPDispatcher;
@@ -372,6 +374,8 @@ public class ControllerModule extends AbstractModule {
// factory to create LoggingRequestHelper instances for LogSearch integration
bind(LoggingRequestHelperFactory.class).to(LoggingRequestHelperFactoryImpl.class);
+ bind(MetricsService.class).to(MetricsServiceImpl.class).in(Scopes.SINGLETON);
+
requestStaticInjection(DatabaseConsistencyCheckHelper.class);
requestStaticInjection(KerberosChecker.class);
requestStaticInjection(AuthorizationHelper.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/AmbariMetricSink.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/AmbariMetricSink.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/AmbariMetricSink.java
new file mode 100644
index 0000000..809176be
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/AmbariMetricSink.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+//package org.apache.hadoop.metrics2.sink.ambari;
+
+package org.apache.ambari.server.metrics.system;
+
+import java.util.Map;
+
+public interface AmbariMetricSink {
+ /**
+ * initialize Collector URI and sink frequency to publish the metrics to AMS
+ **/
+ void init(String protocol, String collectorUri, int frequency);
+
+ /**
+ * Publish metrics to Collector
+ **/
+ void publish(Map<String, Number> metricsMap);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsService.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsService.java
new file mode 100644
index 0000000..23845c9
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsService.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system;
+
+public interface MetricsService extends Runnable {
+ /**
+ * Set up configuration
+ **/
+ void init();
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsSource.java
new file mode 100644
index 0000000..cf10408
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/MetricsSource.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system;
+
+public interface MetricsSource extends Runnable{
+ /**
+ * initialize sink
+ **/
+ void init(AmbariMetricSink sink);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AbstractMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AbstractMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AbstractMetricsSource.java
new file mode 100644
index 0000000..58e2045
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AbstractMetricsSource.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system.impl;
+
+import org.apache.ambari.server.metrics.system.AmbariMetricSink;
+import org.apache.ambari.server.metrics.system.MetricsSource;
+
+import java.util.Map;
+
+public abstract class AbstractMetricsSource implements MetricsSource {
+ protected AmbariMetricSink sink;
+
+ /**
+ * Pass metrics sink to metrics source
+ **/
+ @Override
+ public void init(AmbariMetricSink sink) {
+ this.sink = sink;
+ }
+
+ /**
+ * Get metrics at the instance
+ * @return a map for metrics that maps metrics name to metrics value
+ **/
+ abstract public Map<String, Number> getMetrics();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
new file mode 100644
index 0000000..4618618
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system.impl;
+
+import jline.internal.Log;
+import org.apache.ambari.server.metrics.system.AmbariMetricSink;
+import org.apache.commons.lang.ClassUtils;
+import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements AmbariMetricSink {
+ private static final String APP_ID = "ambari_server";
+ private int timeoutSeconds = 10;
+ private String collectorProtocol;
+ private String collectorUri;
+ private String hostName;
+ private int counter = 0;
+ private int frequency;
+ private List<TimelineMetric> buffer = new ArrayList<>();
+ @Override
+ public void init(String protocol, String collectorUri, int frequency) {
+
+ /**
+ * Protocol is either HTTP or HTTPS, and the collectorURI is the domain name of the collector
+ * An example of the complete collector URI might be: http://c6403.ambari.org/ws/v1/timeline/metrics
+ */
+ this.frequency = frequency;
+ this.collectorProtocol = protocol;
+ this.collectorUri = getCollectorUri(collectorUri);
+
+ try {
+ hostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ Log.info("Error getting host address");
+ }
+ }
+
+ @Override
+ public void publish(Map<String, Number> metricsMap) {
+ List<TimelineMetric> metricsList = createMetricsList(metricsMap);
+
+ if(counter > frequency) {
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ timelineMetrics.setMetrics(buffer);
+
+ String connectUrl = collectorUri;
+ String jsonData = null;
+ try {
+ jsonData = mapper.writeValueAsString(timelineMetrics);
+ } catch (IOException e) {
+ LOG.error("Unable to parse metrics", e);
+ }
+ if (jsonData != null) {
+ emitMetricsJson(connectUrl, jsonData);
+ }
+ counter = 0;
+ } else {
+ buffer.addAll(metricsList);
+ counter++;
+ }
+
+ }
+
+
+ /**
+ * Get a pre-formatted URI for the collector
+ *
+ * @param host
+ */
+ @Override
+ protected String getCollectorUri(String host) {
+ return getCollectorProtocol() + "://" + host + WS_V1_TIMELINE_METRICS;
+ }
+
+ @Override
+ protected String getCollectorProtocol() {
+ return collectorProtocol;
+ }
+
+ @Override
+ protected int getTimeoutSeconds() {
+ return timeoutSeconds;
+ }
+
+ /**
+ * Get the zookeeper quorum for the cluster used to find collector
+ *
+ * @return String "host1:port1,host2:port2"
+ */
+ @Override
+ protected String getZookeeperQuorum() {
+ return null;
+ }
+
+ /**
+ * Get pre-configured list of collectors available
+ *
+ * @return String "host1:port,host2:port"
+ */
+ @Override
+ protected String getConfiguredCollectors() {
+ return null;
+ }
+
+ /**
+ * Get hostname used for calculating write shard.
+ *
+ * @return String "host1"
+ */
+ @Override
+ protected String getHostname() {
+ return hostName;
+ }
+
+ private List<TimelineMetric> createMetricsList(Map<String, Number> metricsMap) {
+ final List<TimelineMetric> metricsList = new ArrayList<>();
+ for (Map.Entry<String, Number> entry : metricsMap.entrySet()) {
+ final long currentTimeMillis = System.currentTimeMillis();
+ String metricsName = entry.getKey();
+ Number value = entry.getValue();
+ TimelineMetric metric = createTimelineMetric(currentTimeMillis, APP_ID, metricsName, value);
+ metricsList.add(metric);
+ }
+ return metricsList;
+ }
+
+ private TimelineMetric createTimelineMetric(long currentTimeMillis, String component, String attributeName,
+ Number attributeValue) {
+ TimelineMetric timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(attributeName);
+ timelineMetric.setHostName(hostName);
+ timelineMetric.setAppId(component);
+ timelineMetric.setStartTime(currentTimeMillis);
+ timelineMetric.setType(ClassUtils.getShortCanonicalName(attributeValue, "Number"));
+ timelineMetric.getMetricValues().put(currentTimeMillis, attributeValue.doubleValue());
+ return timelineMetric;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/Configuration.java
new file mode 100644
index 0000000..705971f
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/Configuration.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system.impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class Configuration {
+ public static final String CONFIG_FILE = "metrics.properties";
+
+ private static Logger LOG = LoggerFactory.getLogger(Configuration.class);
+ private Properties properties;
+
+ public Configuration() {
+ this(readConfigFile());
+ }
+
+ public Configuration(Properties properties) {
+ this.properties = properties;
+ }
+
+ private static Properties readConfigFile() {
+ Properties properties = new Properties();
+
+ //Get property file stream from classpath
+ InputStream inputStream = Configuration.class.getClassLoader().getResourceAsStream(CONFIG_FILE);
+
+ if (inputStream == null) {
+ throw new RuntimeException(CONFIG_FILE + " not found in classpath");
+ }
+
+ // load the properties
+ try {
+ properties.load(inputStream);
+ inputStream.close();
+ } catch (FileNotFoundException fnf) {
+ LOG.info("No configuration file " + CONFIG_FILE + " found in classpath.", fnf);
+ } catch (IOException ie) {
+ throw new IllegalArgumentException("Can't read configuration file " +
+ CONFIG_FILE, ie);
+ }
+
+ return properties;
+ }
+
+ /**
+ * Get the property value for the given key.
+ *
+ * @return the property value
+ */
+ public String getProperty(String key) {
+ return properties.getProperty(key);
+ }
+
+ /**
+ * Get the property value for the given key.
+ *
+ * @return the property value
+ */
+ public String getProperty(String key, String defaultValue) {
+ return properties.getProperty(key, defaultValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
new file mode 100644
index 0000000..a04ca43
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/JvmMetricsSource.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system.impl;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import org.apache.ambari.server.metrics.system.AmbariMetricSink;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Map;
+
+public class JvmMetricsSource extends AbstractMetricsSource {
+ static final MetricRegistry registry = new MetricRegistry();
+ private static Logger LOG = LoggerFactory.getLogger(JvmMetricsSource.class);
+
+ @Override
+ public void init(AmbariMetricSink sink) {
+ super.init(sink);
+ registerAll("gc", new GarbageCollectorMetricSet(), registry);
+ registerAll("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()), registry);
+ registerAll("memory", new MemoryUsageGaugeSet(), registry);
+ registerAll("threads", new ThreadStatesGaugeSet(), registry);
+ }
+
+ @Override
+ public void run() {
+ this.sink.publish(getMetrics());
+ LOG.info("********* Published system metrics to sink **********");
+ }
+
+
+ private void registerAll(String prefix, MetricSet metricSet, MetricRegistry registry) {
+ for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
+ if (entry.getValue() instanceof MetricSet) {
+ registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue(), registry);
+ } else {
+ registry.register(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ public Map<String, Number> getMetrics() {
+ Map<String, Number> map = new HashMap<>();
+ for (String metricName : registry.getGauges().keySet()) {
+ if (metricName.equals("threads.deadlocks") ) continue;
+ Number value = (Number)registry.getGauges().get(metricName).getValue();
+ map.put(metricName, value);
+ }
+ return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
new file mode 100644
index 0000000..1645ebf
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/MetricsServiceImpl.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.metrics.system.impl;
+
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider;
+import org.apache.ambari.server.controller.internal.ServiceConfigVersionResourceProvider;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.metrics.system.AmbariMetricSink;
+import org.apache.ambari.server.metrics.system.MetricsService;
+import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+@Singleton
+public class MetricsServiceImpl implements MetricsService {
+ private static Logger LOG = LoggerFactory.getLogger(MetricsServiceImpl.class);
+ private Map<String, AbstractMetricsSource> sources = new HashMap<>();
+ private AmbariMetricSink sink = new AmbariMetricSinkImpl();
+ private String collectorUri = "";
+ private String collectorProtocol = "";
+ private Configuration configuration;
+
+ @Inject
+ AmbariManagementController amc;
+
+ @Override
+ public void init() {
+ try {
+ configuration = new Configuration();
+ if (collectorUri.isEmpty() || collectorProtocol.isEmpty()) {
+ setCollectorUri();
+ }
+ configureSourceAndSink();
+ } catch (Exception e) {
+ LOG.info("Error initializing MetricsService", e);
+ }
+
+ }
+ @Override
+ public void run() {
+ final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
+ for (Map.Entry<String, AbstractMetricsSource> entry : sources.entrySet()) {
+ publishMetrics(executor, entry);
+ }
+ }
+
+
+ private void setCollectorUri() {
+ InternalAuthenticationToken authenticationToken = new InternalAuthenticationToken("admin");
+ authenticationToken.setAuthenticated(true);
+ SecurityContextHolder.getContext().setAuthentication(authenticationToken);
+ Clusters clusters = amc.getClusters();
+ for (Map.Entry<String, Cluster> kv : clusters.getClusters().entrySet()) {
+ String clusterName = kv.getKey();
+ Resource.Type type = Resource.Type.ServiceConfigVersion;
+
+ Set<String> propertyIds = new HashSet<String>();
+ propertyIds.add(ServiceConfigVersionResourceProvider.SERVICE_CONFIG_VERSION_CONFIGURATIONS_PROPERTY_ID);
+
+ Predicate predicate = new PredicateBuilder().property(
+ ServiceConfigVersionResourceProvider.SERVICE_CONFIG_VERSION_CLUSTER_NAME_PROPERTY_ID).equals(clusterName).and().property(
+ ServiceConfigVersionResourceProvider.SERVICE_CONFIG_VERSION_SERVICE_NAME_PROPERTY_ID).equals("AMBARI_METRICS").and().property(
+ ServiceConfigVersionResourceProvider.SERVICE_CONFIG_VERSION_IS_CURRENT_PROPERTY_ID).equals("true").toPredicate();
+
+ Request request = PropertyHelper.getReadRequest(propertyIds);
+
+ ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(
+ type,
+ PropertyHelper.getPropertyIds(type),
+ PropertyHelper.getKeyPropertyIds(type),
+ amc);
+
+ try {
+ Set<Resource> resources = provider.getResources(request, predicate);
+
+ // get collector uri
+ for (Resource resource : resources) {
+ if (resource != null) {
+ ArrayList<LinkedHashMap<Object, Object>> configs = (ArrayList<LinkedHashMap<Object, Object>>)
+ resource.getPropertyValue(ServiceConfigVersionResourceProvider.SERVICE_CONFIG_VERSION_CONFIGURATIONS_PROPERTY_ID);
+ for (LinkedHashMap<Object, Object> config : configs) {
+ if (config != null && config.get("type").equals("ams-site")) {
+ TreeMap<Object, Object> properties = (TreeMap<Object, Object>) config.get("properties");
+ collectorUri = (String) properties.get("timeline.metrics.service.webapp.address");
+ String which_protocol = (String) properties.get("timeline.metrics.service.http.policy");
+ collectorProtocol = which_protocol.equals("HTTP_ONLY") ? "http" : "https";
+ break;
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.info("Throwing exception when retrieving Collector URI", e);
+ }
+ }
+ }
+
+ private void configureSourceAndSink() {
+ try {
+ LOG.info("********* Configuring Ambari Metrics Sink and Source**********");
+ int frequency = Integer.parseInt(configuration.getProperty("sink.frequency", "10")); // default value 10
+ sink.init(collectorProtocol, collectorUri, frequency);
+ String[] sourceNames = configuration.getProperty("metrics.sources").split(",");
+ for (String sourceName: sourceNames) {
+ String className = configuration.getProperty("source." + sourceName + ".class");
+ Class t = Class.forName(className);
+ AbstractMetricsSource src = (AbstractMetricsSource)t.newInstance();
+ src.init(sink);
+ sources.put(sourceName, src);
+ }
+ }
+ catch (Exception e) {
+ LOG.info("Throwing exception when registering metric sink and source", e);
+ }
+ }
+
+ private void publishMetrics(ScheduledExecutorService executor, Map.Entry<String, AbstractMetricsSource> entry) {
+ String className = entry.getKey();
+ AbstractMetricsSource source = entry.getValue();
+ String interval = "source." + className + ".interval";
+ int duration = Integer.parseInt(configuration.getProperty(interval, "5")); // default value 5
+ try {
+ executor.scheduleWithFixedDelay(source, 0, duration, TimeUnit.SECONDS);
+
+ } catch (Exception e) {
+ LOG.info("Throwing exception when failing scheduling source", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5f9c3ab1/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/default/grafana-ambari-server.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/default/grafana-ambari-server.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/default/grafana-ambari-server.json
new file mode 100644
index 0000000..de66e83
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/default/grafana-ambari-server.json
@@ -0,0 +1,691 @@
+{
+ "id": null,
+ "title": "Ambari Server",
+ "originalTitle": "Ambari Server",
+ "tags": [
+ "ambari"
+ ],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": false,
+ "rows": [
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "25px",
+ "panels": [
+ {
+ "content": "<h4 align=\"center\">Metrics to see the overall status for Ambari Server. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari. You may lose any changes made to this dashboard. If you want to customize, make your own copy.</h6>",
+ "editable": true,
+ "error": false,
+ "id": 10,
+ "isNew": true,
+ "links": [],
+ "mode": "html",
+ "span": 12,
+ "style": {},
+ "title": "",
+ "type": "text"
+ }
+ ],
+ "title": "New row"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 3,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "max",
+ "alias": "MemHeapMax",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "memory.heap.max",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "max",
+ "alias": "MemHeapUsed",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "memory.heap.used",
+ "precision": "default",
+ "refId": "B",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "max",
+ "alias": "MemHeapCommitted",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "memory.heap.committed",
+ "precision": "default",
+ "refId": "C",
+ "seriesAggregator": "none",
+ "transform": "none"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Heap Usage",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "bytes",
+ "short"
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 4,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "max",
+ "alias": "MemNonHeapMax",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "memory.non-heap.max",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "none",
+ "alias": "MemNonHeapUsed",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "memory.non-heap.used",
+ "precision": "default",
+ "refId": "B",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "none",
+ "alias": "MemNonHeapCommitted",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "memory.non-heap.committed",
+ "precision": "default",
+ "refId": "C",
+ "seriesAggregator": "none",
+ "transform": "none"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Off-Heap Usage",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "bytes",
+ "short"
+ ]
+ }
+ ],
+ "showTitle": true,
+ "title": "JVM - Memory Pressure"
+ },
+ {
+ "collapse": true,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 6,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "avg",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "gc.ParNew.count",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "rate"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "GC Count Par New /s",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 5,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "avg",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "gc.ParNew.time",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "rate"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "GC Time Par New /s",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 7,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "max",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "gc.ConcurrentMarkSweep.count",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "rate"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "GC Count CMS /s",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 8,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 6,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "avg",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "gc.ConcurrentMarkSweep.time",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "rate"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "GC Time CMS /s",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ }
+ ],
+ "showTitle": true,
+ "title": "JVM - GC Count"
+ },
+ {
+ "collapse": true,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 9,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "max",
+ "alias": "Total",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "threads.count",
+ "precision": "default",
+ "refId": "A",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "max",
+ "alias": "Daemon",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "threads.daemon.count",
+ "precision": "default",
+ "refId": "B",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "max",
+ "alias": "Deadlock",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "threads.deadlock.count",
+ "precision": "default",
+ "refId": "C",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "max",
+ "alias": "Blocked",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "threads.blocked.count",
+ "precision": "default",
+ "refId": "D",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "max",
+ "alias": "Runnable",
+ "app": "ambari_server",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "threads.runnable.count",
+ "precision": "default",
+ "refId": "E",
+ "seriesAggregator": "none",
+ "transform": "none"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Thread Count",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ }
+ ],
+ "showTitle": true,
+ "title": "JVM - Thread Count"
+ }
+ ],
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": []
+ },
+ "refresh": false,
+ "schemaVersion": 8,
+ "version": 1,
+ "links": []
+}
\ No newline at end of file
[38/55] [abbrv] ambari git commit: AMBARI-18218. File View throws
IllegalArguementException post Ambari 2.4.0.0 upgrade.(gauravn7)
Posted by ol...@apache.org.
AMBARI-18218. File View throws IllegalArguementException post Ambari 2.4.0.0 upgrade.(gauravn7)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74945492
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74945492
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74945492
Branch: refs/heads/branch-dev-logsearch
Commit: 74945492b2a6574eea7d3fb9e93f24c22b98b892
Parents: 967ce4b
Author: Gaurav Nagar <gn...@hortonworks.com>
Authored: Sun Aug 21 15:54:02 2016 +0530
Committer: Gaurav Nagar <gn...@hortonworks.com>
Committed: Sun Aug 21 15:55:12 2016 +0530
----------------------------------------------------------------------
.../org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/74945492/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
index 1c17e6a..d505e9a 100644
--- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/ConfigurationBuilder.java
@@ -21,6 +21,7 @@ package org.apache.ambari.view.utils.hdfs;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.utils.ambari.AmbariApi;
import org.apache.ambari.view.utils.ambari.NoClusterAssociatedException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
@@ -224,7 +225,9 @@ public class ConfigurationBuilder {
private String copyClusterProperty(String propertyName, String instancePropertyName) {
String value = getProperty(HDFS_SITE, propertyName, instancePropertyName);
- conf.set(propertyName, value);
+ if (!StringUtils.isEmpty(value)) {
+ conf.set(propertyName, value);
+ }
LOG.debug("set " + propertyName + " = " + value);
return value;
}
[19/55] [abbrv] ambari git commit: AMBARI-18205. RU,
order Kafka right after Ranger and before HDFS in upgrade pack
(alejandro)
Posted by ol...@apache.org.
AMBARI-18205. RU, order Kafka right after Ranger and before HDFS in upgrade pack (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/125be046
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/125be046
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/125be046
Branch: refs/heads/branch-dev-logsearch
Commit: 125be0469998b608d541e00e1e2ae8b05223ab6d
Parents: a57daad
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Aug 18 16:18:13 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu Aug 18 16:18:13 2016 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 14 +++----
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 14 +++----
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 14 +++----
.../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 30 +++++++--------
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 40 ++++++++++----------
.../services/HIVE/configuration/hive-site.xml | 1 +
.../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 31 +++++++--------
7 files changed, 73 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index 2015440..533a1e9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -118,13 +118,6 @@
<component>ZOOKEEPER_SERVER</component>
</service>
</group>
-
- <group name="KAFKA" title="Kafka">
- <skippable>true</skippable>
- <service name="KAFKA">
- <component>KAFKA_BROKER</component>
- </service>
- </group>
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
@@ -143,6 +136,13 @@
</service>
</group>
+ <group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
+ <service name="KAFKA">
+ <component>KAFKA_BROKER</component>
+ </service>
+ </group>
+
<group name="CORE_MASTER" title="Core Masters">
<service-check>false</service-check>
<service name="HDFS">
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 0611301..4e241f0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -116,13 +116,6 @@
</service>
</group>
- <group name="KAFKA" title="Kafka">
- <skippable>true</skippable>
- <service name="KAFKA">
- <component>KAFKA_BROKER</component>
- </service>
- </group>
-
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -140,6 +133,13 @@
</service>
</group>
+ <group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
+ <service name="KAFKA">
+ <component>KAFKA_BROKER</component>
+ </service>
+ </group>
+
<!-- This group should exist for all RUs that cross a major stack version. -->
<group xsi:type="cluster" name="UPDATE_CLIENT_CONFIGS" title="Update Client Configs">
<direction>UPGRADE</direction>
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 839e544..a256d2f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -126,13 +126,6 @@
</service>
</group>
- <group name="KAFKA" title="Kafka">
- <skippable>true</skippable>
- <service name="KAFKA">
- <component>KAFKA_BROKER</component>
- </service>
- </group>
-
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -150,6 +143,13 @@
</service>
</group>
+ <group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
+ <service name="KAFKA">
+ <component>KAFKA_BROKER</component>
+ </service>
+ </group>
+
<!-- This group should exist for all RUs that cross a major stack version. -->
<group xsi:type="cluster" name="UPDATE_CLIENT_CONFIGS" title="Update Client Configs">
<direction>UPGRADE</direction>
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
index 0374615..abf2db3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
@@ -117,13 +117,6 @@
</service>
</group>
- <group name="KAFKA" title="Kafka">
- <skippable>true</skippable>
- <service name="KAFKA">
- <component>KAFKA_BROKER</component>
- </service>
- </group>
-
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -141,6 +134,13 @@
</service>
</group>
+ <group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
+ <service name="KAFKA">
+ <component>KAFKA_BROKER</component>
+ </service>
+ </group>
+
<group name="CORE_MASTER" title="Core Masters">
<service-check>false</service-check>
<service name="HDFS">
@@ -442,14 +442,6 @@
</component>
</service>
- <service name="KAFKA">
- <component name="KAFKA_BROKER">
- <upgrade>
- <task xsi:type="restart-task" />
- </upgrade>
- </component>
- </service>
-
<service name="RANGER">
<component name="RANGER_ADMIN">
<pre-upgrade>
@@ -511,6 +503,14 @@
</component>
</service>
+ <service name="KAFKA">
+ <component name="KAFKA_BROKER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+
<service name="HDFS">
<component name="NAMENODE">
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index f5d7b18..a4eb513 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -121,13 +121,6 @@
</service>
</group>
- <group name="KAFKA" title="Kafka">
- <skippable>true</skippable>
- <service name="KAFKA">
- <component>KAFKA_BROKER</component>
- </service>
- </group>
-
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -145,6 +138,13 @@
</service>
</group>
+ <group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
+ <service name="KAFKA">
+ <component>KAFKA_BROKER</component>
+ </service>
+ </group>
+
<!-- This group should exist for all RUs that cross a major stack version. -->
<group xsi:type="cluster" name="UPDATE_CLIENT_CONFIGS" title="Update Client Configs">
<direction>UPGRADE</direction>
@@ -529,19 +529,6 @@
</component>
</service>
- <service name="KAFKA">
- <component name="KAFKA_BROKER">
- <pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
- <pre-upgrade>
- <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
- </pre-upgrade>
-
- <upgrade>
- <task xsi:type="restart-task" />
- </upgrade>
- </component>
- </service>
-
<service name="RANGER">
<component name="RANGER_ADMIN">
<pre-downgrade/> <!-- no-op to prevent config changes on downgrade -->
@@ -615,6 +602,19 @@
</component>
</service>
+ <service name="KAFKA">
+ <component name="KAFKA_BROKER">
+ <pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
+ </pre-upgrade>
+
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+
<service name="HDFS">
<component name="NAMENODE">
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-site.xml
index bf1de39..cd19519 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-site.xml
@@ -22,5 +22,6 @@ limitations under the License.
<name>atlas.rest.address</name>
<value>http://localhost:21000</value>
<deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/125be046/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
index 1ce75d9..60e81c0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
@@ -122,13 +122,6 @@
</service>
</group>
- <group name="KAFKA" title="Kafka">
- <skippable>true</skippable>
- <service name="KAFKA">
- <component>KAFKA_BROKER</component>
- </service>
- </group>
-
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -147,6 +140,14 @@
</service>
</group>
+
+ <group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
+ <service name="KAFKA">
+ <component>KAFKA_BROKER</component>
+ </service>
+ </group>
+
<group name="CORE_MASTER" title="Core Masters">
<service-check>false</service-check>
<service name="HDFS">
@@ -478,14 +479,6 @@
</component>
</service>
- <service name="KAFKA">
- <component name="KAFKA_BROKER">
- <upgrade>
- <task xsi:type="restart-task" />
- </upgrade>
- </component>
- </service>
-
<service name="RANGER">
<component name="RANGER_ADMIN">
<pre-upgrade>
@@ -553,6 +546,14 @@
</component>
</service>
+ <service name="KAFKA">
+ <component name="KAFKA_BROKER">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+
<service name="HDFS">
<component name="NAMENODE">
<upgrade>
[02/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
new file mode 100644
index 0000000..de8f8e3
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
@@ -0,0 +1,2234 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.2",
+ "target-version": "2.3",
+ "options": {
+ "config-types": {
+ "falcon-startup.properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "FALCON"
+ ]
+ },
+ "tez-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "TEZ"
+ ]
+ },
+ "oozie-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "OOZIE"
+ ]
+ },
+ "hbase-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "HBASE"
+ ]
+ },
+ "mapred-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "MAPREDUCE2"
+ ]
+ },
+ "hdfs-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "HDFS"
+ ]
+ },
+ "yarn-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "YARN"
+ ]
+ },
+ "capacity-scheduler": {
+ "merged-copy": "yes",
+ "required-services": [
+ "YARN"
+ ]
+ },
+ "kafka-broker": {
+ "merged-copy": "yes",
+ "required-services": [
+ "KAFKA"
+ ]
+ },
+ "ranger-hdfs-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger-hdfs-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger-hdfs-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger-hdfs-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger-hbase-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger-hbase-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger-hbase-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger-hbase-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger-hive-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "ranger-hive-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "ranger-hive-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "ranger-hive-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "ranger-knox-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "ranger-knox-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "ranger-knox-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "ranger-knox-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "ranger-storm-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "ranger-storm-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "ranger-storm-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "ranger-storm-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "ranger-kafka-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KAFKA"
+ ]
+ },
+ "ranger-kafka-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KAFKA"
+ ]
+ },
+ "ranger-kafka-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KAFKA"
+ ]
+ },
+ "ranger-kafka-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "KAFKA"
+ ]
+ },
+ "ranger-yarn-policymgr-ssl": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "YARN"
+ ]
+ },
+ "ranger-yarn-security": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "YARN"
+ ]
+ },
+ "ranger-yarn-audit": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "YARN"
+ ]
+ },
+ "ranger-yarn-plugin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER",
+ "YARN"
+ ]
+ },
+ "kafka-env": {
+ "merged-copy": "yes",
+ "required-services": [
+ "KAFKA"
+ ]
+ },
+ "yarn-env": {
+ "merged-copy": "yes",
+ "required-services": [
+ "YARN"
+ ]
+ },
+ "admin-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "usersync-properties": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger-env": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger-admin-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger-ugsync-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "storm-env": {
+ "merged-copy": "yes",
+ "required-services": [
+ "STORM"
+ ]
+ },
+ "webhcat-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "HIVE"
+ ]
+ },
+ "storm-cluster-log4j": {
+ "merged-copy": "yes",
+ "required-services": [
+ "STORM"
+ ]
+ },
+ "storm-worker-log4j": {
+ "merged-copy": "yes",
+ "required-services": [
+ "STORM"
+ ]
+ }
+ }
+ },
+ "properties": {
+ "storm-env": {
+ "nimbus_seeds_supported": "true",
+ "storm_logs_supported": "true"
+ },
+ "storm-cluster-log4j": {
+ "content": "\n \n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n\n<configuration monitorInterval=\"60\">\n<properties>\n <property name=\"pattern\">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property>\n <property name=\"patt
ernMetris\">%d %-8r %m%n</property>\n</properties>\n<appenders>\n <RollingFile name=\"A1\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.%i\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile name=\"ACCESS\"\n fileName=\"${sys:storm.log.dir}/access.log\"\n filePattern=\"${sys:storm.log.dir}/access.log.%i\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile
name=\"METRICS\"\n fileName=\"${sys:storm.log.dir}/metrics.log\"\n filePattern=\"${sys:storm.log.dir}/metrics.log.%i\">\n <PatternLayout>\n <pattern>${patternMetris}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"2 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <Syslog name=\"syslog\" format=\"RFC5424\" host=\"localhost\" port=\"514\"\n protocol=\"UDP\" appName=\"[${sys:daemon.name}]\" mdcId=\"mdc\" includeMDC=\"true\"\n facility=\"LOCAL5\" enterpriseNumber=\"18060\" newLine=\"true\" exceptionPattern=\"%rEx{full}\"\n messageId=\"[${sys:user.name}:S0]\" id=\"storm\"/>\n</appenders>\n<loggers>\n\n <Logger name=\"backtype.storm.security.auth.authorizer\" level=\"info\">\n <AppenderRef ref=\"ACCESS\"/>\n </Logger>\n <Logger name=\"backtype.storm.metric.Loggi
ngMetricsConsumer\" level=\"info\">\n <AppenderRef ref=\"METRICS\"/>\n </Logger>\n <root level=\"info\"> <!-- We log everything -->\n <appender-ref ref=\"A1\"/>\n <appender-ref ref=\"syslog\"/>\n </root>\n</loggers>\n</configuration>\n \n "
+ },
+ "storm-worker-log4j": {
+ "content": "\n \n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n\n<configuration monitorInterval=\"60\">\n<properties>\n <property name=\"pattern\">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property>\n <property name=\"patt
ernNoTime\">%msg%n</property>\n</properties>\n<appenders>\n <RollingFile name=\"A1\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.%i.gz\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile name=\"STDOUT\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}.out\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.out.%i.gz\">\n <PatternLayout>\n <pattern>${patternNoTime}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"4\"/>\n
</RollingFile>\n <RollingFile name=\"STDERR\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}.err\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.err.%i.gz\">\n <PatternLayout>\n <pattern>${patternNoTime}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"4\"/>\n </RollingFile>\n <Syslog name=\"syslog\" format=\"RFC5424\" host=\"localhost\" port=\"514\"\n protocol=\"UDP\" appName=\"[${sys:storm.id}:${sys:worker.port}]\" mdcId=\"mdc\" includeMDC=\"true\"\n facility=\"LOCAL5\" enterpriseNumber=\"18060\" newLine=\"true\" exceptionPattern=\"%rEx{full}\"\n messageId=\"[${sys:user.name}:${sys:logging.sensitivity}]\" id=\"storm\"/>\n</appenders>\n<loggers>\n <root level=\"info\"> <!-- We log everything -->\n <appender-ref ref=\"A1\"/>\n
<appender-ref ref=\"syslog\"/>\n </root>\n <Logger name=\"STDERR\" level=\"INFO\">\n <appender-ref ref=\"STDERR\"/>\n <appender-ref ref=\"syslog\"/>\n </Logger>\n <Logger name=\"STDOUT\" level=\"INFO\">\n <appender-ref ref=\"STDOUT\"/>\n <appender-ref ref=\"syslog\"/>\n </Logger>\n</loggers>\n</configuration>\n \n "
+ },
+ "falcon-startup.properties": {
+ "*.shared.libs": "activemq-core,ant,geronimo-j2ee-management,jms,json-simple,oozie-client,spring-jms,commons-lang3,commons-el",
+ "*.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap,\\\n org.apache.falcon.service.SharedLibraryHostingService"
+ },
+ "tez-site": {
+ "tez.am.view-acls": "*",
+ "tez.tez-ui.history-url.base": {
+ "value": "{TEZ_HISTORY_URL_BASE}",
+ "template": "yes"
+ }
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.AuthorizationService.authorization.enabled": "true",
+ "oozie.authentication.kerberos.name.rules": "RULE:[2:$1@$0]([jt]t@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-MAPREDUSER/\nRULE:[2:$1@$0]([nd]n@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HDFSUSER/\nRULE:[2:$1@$0](hm@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\nRULE:[2:$1@$0](rs@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\nDEFAULT",
+ "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials",
+ "oozie.service.CallableQueueService.callable.concurrency": {
+ "remove": "yes"
+ },
+ "oozie.service.CallableQueueService.queue.size": {
+ "remove": "yes"
+ },
+ "oozie.service.CallableQueueService.threads": {
+ "remove": "yes"
+ },
+ "oozie.service.JPAService.create.db.schema": {
+ "remove": "yes"
+ },
+ "oozie.service.JPAService.pool.max.active.conn": {
+ "remove": "yes"
+ },
+ "oozie.service.PurgeService.older.than": {
+ "remove": "yes"
+ },
+ "oozie.service.PurgeService.purge.interval": {
+ "remove": "yes"
+ },
+ "oozie.service.SchemaService.wf.ext.schemas": {
+ "remove": "yes"
+ },
+ "oozie.service.WorkflowAppService.system.libpath": {
+ "remove": "yes"
+ },
+ "oozie.service.coord.normal.default.timeout": {
+ "remove": "yes"
+ },
+ "oozie.service.coord.push.check.requeue.interval": {
+ "remove": "yes"
+ },
+ "oozie.services": {
+ "remove": "yes"
+ },
+ "oozie.system.id": {
+ "remove": "yes"
+ },
+ "oozie.systemmode": {
+ "remove": "yes"
+ },
+ "use.system.libpath.for.mapreduce.and.pig.jobs": {
+ "remove": "yes"
+ }
+ },
+ "hbase-site": {
+ "hbase.region.server.rpc.scheduler.factory.class": {
+ "value": "{HBASE_REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS}",
+ "required-services": [
+ "HBASE"
+ ],
+ "template": "yes"
+ },
+ "hbase.rpc.controllerfactory.class": {
+ "value": "{HBASE_RPC_CONTROLLERFACTORY_CLASS}",
+ "template": "yes",
+ "required-services": [
+ "HBASE"
+ ]
+ },
+ "hbase.regionserver.wal.codec": {
+ "value": "{HBASE_REGIONSERVER_WAL_CODEC}",
+ "template": "yes",
+ "required-services": [
+ "HBASE"
+ ]
+ },
+ "phoenix.functions.allowUserDefinedFunctions": "true",
+ "fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
+ "hbase.bucketcache.percentage.in.combinedcache": {
+ "remove": "yes"
+ },
+ "hbase.coprocessor.master.classes": {
+ "value": "{HBASE_COPROCESS_MASTER_CLASSES}",
+ "template": "yes",
+ "can-create": "no",
+ "required-services": [
+ "HBASE",
+ "RANGER"
+ ]
+ },
+ "hbase.coprocessor.region.classes": {
+ "value": "{HBASE_COPROCESSOR_REGION_CLASSES}",
+ "template": "yes",
+ "can-create": "no",
+ "required-services": [
+ "HBASE",
+ "RANGER"
+ ]
+ }
+ },
+ "mapred-site": {
+ "mapreduce.fileoutputcommitter.algorithm.version": "1",
+ "mapreduce.task.tmp.dir": {
+ "remove": "yes"
+ }
+ },
+ "hdfs-site": {
+ "dfs.namenode.inode.attributes.provider.class": {
+ "value": "org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer",
+ "required-services": [
+ "RANGER"
+ ]
+ }
+ },
+ "yarn-site": {
+ "yarn.node-labels.enabled": {
+ "value": "false",
+ "override": "no"
+ },
+ "yarn.timeline-service.recovery.enabled": "true",
+ "yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
+ "yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
+ },
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.root.default-node-label-expression": "",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {
+ "remove": "yes"
+ },
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {
+ "remove": "yes"
+ }
+ },
+ "kafka-broker": {
+ "listeners": {
+ "value": "{KAFKA_LISTENERS}",
+ "template": "yes"
+ },
+ "controlled.shutdown.enable": "true"
+ },
+ "ranger-hdfs-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore.credential.file": {
+ "value": "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "xasecure.policymgr.clientssl.truststore.credential.file": {
+ "value": "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ }
+ },
+ "ranger-hdfs-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.hdfs.policy.source.impl": {
+ "value": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger.plugin.hdfs.policy.rest.ssl.config.file": {
+ "value": "/usr/hdp/current/hadoop-client/conf/ranger-policymgr-ssl.xml",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger.plugin.hdfs.policy.pollIntervalMs": {
+ "value": "30000",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "ranger.plugin.hdfs.policy.cache.dir": {
+ "value": "{RANGER_PLUGIN_HDFS_POLICY_CACHE_DIR}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "xasecure.add-hadoop-authorization": {
+ "value": "true",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ }
+ },
+ "ranger-hdfs-audit": {
+ "xasecure.audit.credential.provider.file": {
+ "value": "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.solr": {
+ "value": "false",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.solr.urls": {
+ "value": "{{ranger_audit_solr_urls}}",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.solr.zookeepers": {
+ "value": "none",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.solr.batch.filespool.dir": {
+ "value": "/var/log/hadoop/hdfs/audit/solr/spool",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.db.jdbc.driver": {
+ "value": "{HDFS_JDBC_DRIVER}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.db.jdbc.url": {
+ "value": "{HDFS_AUDIT_JDBC_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.provider.summary.enabled": {
+ "value": "false",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hadoop/hdfs/audit/db/spool",
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db.password": {
+ "value": "{{xa_audit_db_password}}"
+ },
+ "xasecure.audit.destination.db": {
+ "value": "{AUDIT_TO_DB_HDFS}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.hdfs": {
+ "value": "{AUDIT_TO_HDFS_HDFS}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": {
+ "value": "{AUDIT_HDFS_FILESPOOL_DIR_HDFS}",
+ "template": "yes"
+ }
+ },
+ "ranger-hdfs-plugin-properties": {
+ "XAAUDIT.HDFS.DESTINTATION_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ }
+ },
+ "ranger-hbase-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore.credential.file": {
+ "value": "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.policymgr.clientssl.truststore.credential.file": {
+ "value": "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ }
+ },
+ "ranger-hbase-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.hbase.policy.source.impl": {
+ "value": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger.plugin.hbase.policy.rest.ssl.config.file": {
+ "value": "/usr/hdp/current/hbase-client/conf/ranger-policymgr-ssl.xml",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger.plugin.hbase.policy.pollIntervalMs": {
+ "value": "30000",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "ranger.plugin.hbase.policy.cache.dir": {
+ "value": "{RANGER_PLUGIN_HBASE_POLICY_CACHE_DIR}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ }
+ },
+ "ranger-hbase-audit": {
+ "xasecure.audit.credential.provider.file": {
+ "value": "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.solr": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.solr.urls": {
+ "value": "{{ranger_audit_solr_urls}}"
+ },
+ "xasecure.audit.destination.solr.zookeepers": {
+ "value": "none"
+ },
+ "xasecure.audit.destination.solr.batch.filespool.dir": {
+ "value": "/var/log/hbase/audit/solr/spool"
+ },
+ "xasecure.audit.destination.db.jdbc.driver": {
+ "value": "{HBASE_JDBC_DRIVER}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.db.jdbc.url": {
+ "value": "{HBASE_AUDIT_JDBC_URL}",
+ "template": "yes"
+ },
+ "xasecure.audit.provider.summary.enabled": {
+ "value": "true"
+ },
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hbase/audit/db/spool",
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db.password": {
+ "value": "{{xa_audit_db_password}}"
+ },
+ "xasecure.audit.destination.db": {
+ "value": "{AUDIT_TO_DB_HBASE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs": {
+ "value": "{AUDIT_TO_HDFS_HBASE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": {
+ "value": "{AUDIT_HDFS_FILESPOOL_DIR_HBASE}",
+ "template": "yes"
+ }
+ },
+ "ranger-hbase-plugin-properties": {
+ "XAAUDIT.HDFS.DESTINTATION_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "SQL_CONNECTOR_JAR": {
+ "remove": "yes"
+ }
+ },
+ "ranger-hive-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore.credential.file": {
+ "value": "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.policymgr.clientssl.truststore.credential.file": {
+ "value": "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ }
+ },
+ "ranger-hive-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.hive.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "ranger.plugin.hive.policy.rest.ssl.config.file": "/usr/hdp/current/hive-server2/conf/conf.server/ranger-policymgr-ssl.xml",
+ "ranger.plugin.hive.policy.pollIntervalMs": "30000",
+ "ranger.plugin.hive.policy.cache.dir": {
+ "value": "{RANGER_PLUGIN_HIVE_POLICY_CACHE_DIR}",
+ "template": "yes"
+ }
+ },
+ "ranger-hive-audit": {
+ "xasecure.audit.credential.provider.file": {
+ "value": "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.solr": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.solr.urls": {
+ "value": "{{ranger_audit_solr_urls}}"
+ },
+ "xasecure.audit.destination.solr.zookeepers": {
+ "value": "none"
+ },
+ "xasecure.audit.destination.solr.batch.filespool.dir": {
+ "value": "/var/log/hive/audit/solr/spool"
+ },
+ "xasecure.audit.destination.db.jdbc.driver": {
+ "value": "{HIVE_JDBC_DRIVER}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.db.jdbc.url": {
+ "value": "{HIVE_AUDIT_JDBC_URL}",
+ "template": "yes"
+ },
+ "xasecure.audit.provider.summary.enabled": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hive/audit/db/spool",
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db.password": {
+ "value": "{{xa_audit_db_password}}"
+ },
+ "xasecure.audit.destination.db": {
+ "value": "{AUDIT_TO_DB_HIVE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs": {
+ "value": "{AUDIT_TO_HDFS_HIVE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": {
+ "value": "{AUDIT_HDFS_FILESPOOL_DIR_HIVE}",
+ "template": "yes"
+ }
+ },
+ "ranger-hive-plugin-properties": {
+ "XAAUDIT.HDFS.DESTINTATION_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ }
+ },
+ "ranger-knox-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore.credential.file": {
+ "value": "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.policymgr.clientssl.truststore.credential.file": {
+ "value": "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ }
+ },
+ "ranger-knox-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.knox.policy.source.impl": {
+ "value": "org.apache.ranger.admin.client.RangerAdminJersey2RESTClient"
+ },
+ "ranger.plugin.knox.policy.rest.ssl.config.file": "/usr/hdp/current/knox-server/conf/ranger-policymgr-ssl.xml",
+ "ranger.plugin.knox.policy.pollIntervalMs": "30000",
+ "ranger.plugin.knox.policy.cache.dir": {
+ "value": "{RANGER_PLUGIN_KNOX_POLICY_CACHE_DIR}",
+ "template": "yes"
+ }
+ },
+ "ranger-knox-audit": {
+ "xasecure.audit.credential.provider.file": {
+ "value": "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.solr": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.solr.urls": {
+ "value": "{{ranger_audit_solr_urls}}"
+ },
+ "xasecure.audit.destination.solr.zookeepers": {
+ "value": "none"
+ },
+ "xasecure.audit.destination.solr.batch.filespool.dir": {
+ "value": "/var/log/knox/audit/solr/spool"
+ },
+ "xasecure.audit.destination.db.jdbc.driver": {
+ "value": "{KNOX_JDBC_DRIVER}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.db.jdbc.url": {
+ "value": "{KNOX_AUDIT_JDBC_URL}",
+ "template": "yes"
+ },
+ "xasecure.audit.provider.summary.enabled": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/knox/audit/db/spool",
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db.password": {
+ "value": "{{xa_audit_db_password}}"
+ },
+ "xasecure.audit.destination.db": {
+ "value": "{AUDIT_TO_DB_KNOX}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs": {
+ "value": "{AUDIT_TO_HDFS_KNOX}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": {
+ "value": "{AUDIT_HDFS_FILESPOOL_DIR_KNOX}",
+ "template": "yes"
+ }
+ },
+ "ranger-knox-plugin-properties": {
+ "XAAUDIT.HDFS.DESTINTATION_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ }
+ },
+ "ranger-storm-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore.credential.file": {
+ "value": "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.policymgr.clientssl.truststore.credential.file": {
+ "value": "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ }
+ },
+ "ranger-storm-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.storm.policy.source.impl": {
+ "value": "org.apache.ranger.admin.client.RangerAdminRESTClient"
+ },
+ "ranger.plugin.storm.policy.rest.ssl.config.file": "/usr/hdp/current/storm-client/conf/ranger-policymgr-ssl.xml",
+ "ranger.plugin.storm.policy.pollIntervalMs": "30000",
+ "ranger.plugin.storm.policy.cache.dir": {
+ "value": "{RANGER_PLUGIN_STORM_POLICY_CACHE_DIR}",
+ "template": "yes"
+ }
+ },
+ "ranger-storm-audit": {
+ "xasecure.audit.credential.provider.file": {
+ "value": "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.solr": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.solr.urls": {
+ "value": "{{ranger_audit_solr_urls}}"
+ },
+ "xasecure.audit.destination.solr.zookeepers": {
+ "value": "none"
+ },
+ "xasecure.audit.destination.solr.batch.filespool.dir": {
+ "value": "/var/log/storm/audit/solr/spool"
+ },
+ "xasecure.audit.destination.db.jdbc.driver": {
+ "value": "{STORM_JDBC_DRIVER}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.db.jdbc.url": {
+ "value": "{STORM_AUDIT_JDBC_URL}",
+ "template": "yes"
+ },
+ "xasecure.audit.provider.summary.enabled": {
+ "value": "false"
+ },
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/storm/audit/db/spool",
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db.password": {
+ "value": "{{xa_audit_db_password}}"
+ },
+ "xasecure.audit.destination.db": {
+ "value": "{AUDIT_TO_DB_STORM}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs": {
+ "value": "AUDIT_TO_HDFS_STORM",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": {
+ "value": "{AUDIT_HDFS_FILESPOOL_DIR_STORM}",
+ "template": "yes"
+ }
+ },
+ "ranger-storm-plugin-properties": {
+ "XAAUDIT.HDFS.DESTINTATION_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": {
+ "remove": "yes"
+ },
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": {
+ "remove": "yes"
+ }
+ },
+ "ranger-kafka-plugin-properties": {
+ "policy_user": "ambari-qa",
+ "hadoop.rpc.protection": "",
+ "common.name.for.certificate": "",
+ "zookeeper.connect": "localhost:2181",
+ "ranger-kafka-plugin-enabled": "No",
+ "REPOSITORY_CONFIG_USERNAME": "kafka",
+ "REPOSITORY_CONFIG_PASSWORD": "kafka"
+ },
+ "ranger-kafka-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore": "/usr/hdp/current/kafka-broker/config/ranger-plugin-keystore.jks",
+ "xasecure.policymgr.clientssl.keystore.password": "myKeyFilePassword",
+ "xasecure.policymgr.clientssl.truststore": "/usr/hdp/current/kafka-broker/config/ranger-plugin-truststore.jks",
+ "xasecure.policymgr.clientssl.truststore.password": "changeit",
+ "xasecure.policymgr.clientssl.keystore.credential.file": "jceks://file/{{credential_file}}",
+ "xasecure.policymgr.clientssl.truststore.credential.file": "jceks://file/{{credential_file}}"
+ },
+ "ranger-kafka-audit": {
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db": "false",
+ "xasecure.audit.destination.db.jdbc.url": "{{audit_jdbc_url}}",
+ "xasecure.audit.destination.db.user": "{{xa_audit_db_user}}",
+ "xasecure.audit.destination.db.password": "crypted",
+ "xasecure.audit.destination.db.jdbc.driver": "{{jdbc_driver}}",
+ "xasecure.audit.credential.provider.file": "jceks://file{{credential_file}}",
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/kafka/audit/db/spool",
+ "xasecure.audit.destination.hdfs": "true",
+ "xasecure.audit.destination.hdfs.dir": "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit",
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": "/var/log/kafka/audit/hdfs/spool",
+ "xasecure.audit.destination.solr": "true",
+ "xasecure.audit.destination.solr.urls": "{{ranger_audit_solr_urls}}",
+ "xasecure.audit.destination.solr.zookeepers": "none",
+ "xasecure.audit.destination.solr.batch.filespool.dir": "/var/log/kafka/audit/solr/spool",
+ "xasecure.audit.provider.summary.enabled": "true"
+ },
+ "ranger-kafka-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.kafka.service.name": "{{repo_name}}",
+ "ranger.plugin.kafka.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "ranger.plugin.kafka.policy.rest.url": "{{policymgr_mgr_url}}",
+ "ranger.plugin.kafka.policy.rest.ssl.config.file": "/etc/kafka/conf/ranger-policymgr-ssl.xml",
+ "ranger.plugin.kafka.policy.pollIntervalMs": "30000",
+ "ranger.plugin.kafka.policy.cache.dir": "/etc/ranger/{{repo_name}}/policycache"
+ },
+ "kafka-env": {
+ "is_supported_kafka_ranger": "true"
+ },
+ "ranger-yarn-plugin-properties": {
+ "policy_user": "ambari-qa",
+ "hadoop.rpc.protection": "",
+ "common.name.for.certificate": "",
+ "ranger-yarn-plugin-enabled": "No",
+ "REPOSITORY_CONFIG_USERNAME": "yarn",
+ "REPOSITORY_CONFIG_PASSWORD": "yarn"
+ },
+ "ranger-yarn-policymgr-ssl": {
+ "xasecure.policymgr.clientssl.keystore": "/usr/hdp/current/hadoop-client/conf/ranger-yarn-plugin-keystore.jks",
+ "xasecure.policymgr.clientssl.keystore.password": "myKeyFilePassword",
+ "xasecure.policymgr.clientssl.truststore": "/usr/hdp/current/hadoop-client/conf/ranger-yarn-plugin-truststore.jks",
+ "xasecure.policymgr.clientssl.truststore.password": "changeit",
+ "xasecure.policymgr.clientssl.keystore.credential.file": "jceks://file/{{credential_file}}",
+ "xasecure.policymgr.clientssl.truststore.credential.file": "jceks://file/{{credential_file}}"
+ },
+ "ranger-yarn-audit": {
+ "xasecure.audit.is.enabled": "true",
+ "xasecure.audit.destination.db": "false",
+ "xasecure.audit.destination.db.jdbc.url": "{{audit_jdbc_url}}",
+ "xasecure.audit.destination.db.user": "{{xa_audit_db_user}}",
+ "xasecure.audit.destination.db.password": "crypted",
+ "xasecure.audit.destination.db.jdbc.driver": "{{jdbc_driver}}",
+ "xasecure.audit.credential.provider.file": "jceks://file{{credential_file}}",
+ "xasecure.audit.destination.db.batch.filespool.dir": "/var/log/hadoop/yarn/audit/db/spool",
+ "xasecure.audit.destination.hdfs": "true",
+ "xasecure.audit.destination.hdfs.dir": "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit",
+ "xasecure.audit.destination.hdfs.batch.filespool.dir": "/var/log/hadoop/yarn/audit/hdfs/spool",
+ "xasecure.audit.destination.solr": "false",
+ "xasecure.audit.destination.solr.urls": "{{ranger_audit_solr_urls}}",
+ "xasecure.audit.destination.solr.zookeepers": "none",
+ "xasecure.audit.destination.solr.batch.filespool.dir": "/var/log/hadoop/yarn/audit/solr/spool",
+ "xasecure.audit.provider.summary.enabled": "false"
+ },
+ "ranger-yarn-security": {
+ "upgrade.transition": "yes",
+ "ranger.plugin.yarn.service.name": "{{repo_name}}",
+ "ranger.plugin.yarn.policy.source.impl": "org.apache.ranger.admin.client.RangerAdminRESTClient",
+ "ranger.plugin.yarn.policy.rest.url": "{{policymgr_mgr_url}}",
+ "ranger.plugin.yarn.policy.rest.ssl.config.file": "/usr/hdp/current/hadoop-client/conf/ranger-policymgr-ssl-yarn.xml",
+ "ranger.plugin.yarn.policy.pollIntervalMs": "30000",
+ "ranger.plugin.yarn.policy.cache.dir": "/etc/ranger/{{repo_name}}/policycache"
+ },
+ "yarn-env": {
+ "is_supported_yarn_ranger": "true"
+ },
+ "admin-properties": {
+ "policymgr_http_enabled": {
+ "remove": "yes"
+ },
+ "SQL_COMMAND_INVOKER": {
+ "remove": "yes"
+ }
+ },
+ "ranger-env": {
+ "oracle_home": {
+ "remove": "yes"
+ },
+ "create_db_dbuser": {
+ "value": "true"
+ },
+ "xml_configurations_supported": {
+ "value": "true"
+ },
+ "ranger_privelege_user_jdbc_url": {
+ "value": "{RANGER_ROOT_JDBC_URL}",
+ "template": "yes"
+ },
+ "ranger-yarn-plugin-enabled": {
+ "value": "No"
+ },
+ "ranger-kafka-plugin-enabled":{
+ "value": "No"
+ },
+ "xasecure.audit.destination.solr":{
+ "value": "false"
+ },
+ "is_solrCloud_enabled":{
+ "value": "false"
+ },
+ "xasecure.audit.destination.hdfs": {
+ "value": "{AUDIT_TO_HDFS}",
+ "template": "yes"
+ },
+ "xasecure.audit.destination.db": {
+ "value": "{AUDIT_TO_DB}",
+ "template": "yes"
+ }
+ },
+ "ranger-admin-site": {
+ "upgrade.transition": "yes",
+ "ranger.service.host": {
+ "value": "{RANGER_HOST}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.service.https.attrib.ssl.enabled": {
+ "value": "false"
+ },
+ "ranger.jpa.jdbc.driver": {
+ "value": "{RANGER_JDBC_DRIVER}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.jpa.jdbc.url": {
+ "value": "{RANGER_JDBC_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.jpa.jdbc.credential.alias": {
+ "value": "rangeradmin"
+ },
+ "ranger.credential.provider.path": {
+ "value": "/etc/ranger/admin/rangeradmin.jceks"
+ },
+ "ranger.audit.source.type": {
+ "value": "db"
+ },
+ "ranger.audit.solr.urls": {
+ "value": "http://solr_host:6083/solr/ranger_audits"
+ },
+ "ranger.jpa.audit.jdbc.driver": {
+ "value": "{RANGER_JDBC_DRIVER}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.jpa.audit.jdbc.url": {
+ "value": "{RANGER_AUDIT_JDBC_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.jpa.audit.jdbc.credential.alias": {
+ "value": "rangeraudit"
+ },
+ "ranger.jpa.jdbc.dialect": {
+ "value": "{RANGER_JDBC_DIALECT}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.jpa.audit.jdbc.dialect": {
+ "value": "{RANGER_JDBC_DIALECT}",
+ "template": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "ranger.audit.solr.zookeepers": {
+ "value": "NONE"
+ },
+ "ranger.audit.solr.username": {
+ "value": "ranger_solr"
+ },
+ "ranger.audit.solr.password": {
+ "value": "NONE"
+ },
+ "ranger.externalurl": {
+ "value": "{{ranger_external_url}}"
+ },
+ "ranger.jpa.jdbc.user": {
+ "value": "{{ranger_db_user}}"
+ },
+ "ranger.jpa.audit.jdbc.user": {
+ "value": "{{ranger_audit_db_user}}"
+ },
+ "ranger.jpa.jdbc.password": {
+ "value": "_"
+ },
+ "ranger.jpa.audit.jdbc.password": {
+ "value": "_"
+ },
+ "ranger.sso.providerurl": {
+ "value": ""
+ },
+ "ranger.sso.publicKey": {
+ "value": ""
+ },
+ "ranger.sso.cookiename": {
+ "value": "hadoop-jwt"
+ },
+ "ranger.sso.enabled": {
+ "value": "false"
+ },
+ "ranger.sso.query.param.originalurl": {
+ "value": "originalUrl"
+ },
+ "ranger.sso.browser.useragent": {
+ "value": "Mozilla,chrome"
+ },
+ "ranger.ldap.user.searchfilter":{
+ "value": "{{ranger_ug_ldap_user_searchfilter}}"
+ },
+ "ranger.ldap.group.roleattribute":{
+ "value": "cn"
+ },
+ "ranger.ldap.base.dn":{
+ "value": "dc=example,dc=com"
+ },
+ "ranger.ldap.bind.dn":{
+ "value": "{{ranger_ug_ldap_bind_dn}}"
+ },
+ "ranger.ldap.bind.password":{
+ "value": "{{ranger_usersync_ldap_ldapbindpassword}}"
+ },
+ "ranger.ldap.referral":{
+ "value": "ignore"
+ },
+ "ranger.ldap.ad.base.dn":{
+ "value": "dc=example,dc=com"
+ },
+ "ranger.ldap.ad.bind.dn":{
+ "value": "{{ranger_ug_ldap_bind_dn}}"
+ },
+ "ranger.ldap.ad.bind.password":{
+ "value": "{{ranger_usersync_ldap_ldapbindpassword}}"
+ },
+ "ranger.ldap.ad.user.searchfilter":{
+ "value": "{{ranger_ug_ldap_user_searchfilter}}"
+ },
+ "ranger.ldap.ad.referral":{
+ "value": "ignore"
+ }
+ },
+ "ranger-ugsync-site": {
+ "upgrade.transition": "yes",
+ "ranger.usersync.port": {
+ "value": "5151"
+ },
+ "ranger.usersync.ssl": {
+ "value": "true"
+ },
+ "ranger.usersync.keystore.file": {
+ "value": "/usr/hdp/current/ranger-usersync/conf/unixauthservice.jks"
+ },
+ "ranger.usersync.keystore.password": {
+ "value": "UnIx529p"
+ },
+ "ranger.usersync.truststore.file": {
+ "value": "/usr/hdp/current/ranger-usersync/conf/mytruststore.jks"
+ },
+ "ranger.usersync.truststore.password": {
+ "value": "changeit"
+ },
+ "ranger.usersync.passwordvalidator.path": {
+ "value": "./native/credValidator.uexe"
+ },
+ "ranger.usersync.enabled": {
+ "value": "true"
+ },
+ "ranger.usersync.sink.impl.class": {
+ "value": "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder"
+ },
+ "ranger.usersync.policymanager.maxrecordsperapicall": {
+ "value": "1000"
+ },
+ "ranger.usersync.policymanager.mockrun": {
+ "value": "false"
+ },
+ "ranger.usersync.filesource.file": {
+ "value": "/tmp/usergroup.txt"
+ },
+ "ranger.usersync.filesource.text.delimiter": {
+ "value": ","
+ },
+ "ranger.usersync.ldap.bindalias": {
+ "value": "ranger.usersync.ldap.bindalias"
+ },
+ "ranger.usersync.ldap.bindkeystore": {
+ "value": ""
+ },
+ "ranger.usersync.group.searchenabled": {
+ "value": ""
+ },
+ "ranger.usersync.group.usermapsyncenabled": {
+ "value": ""
+ },
+ "ranger.usersync.group.searchbase": {
+ "value": ""
+ },
+ "ranger.usersync.group.searchscope": {
+ "value": ""
+ },
+ "ranger.usersync.group.objectclass": {
+ "value": ""
+ },
+ "ranger.usersync.group.searchfilter": {
+ "value": "empty"
+ },
+ "ranger.usersync.group.nameattribute": {
+ "value": ""
+ },
+ "ranger.usersync.ldap.searchBase": {
+ "value": ""
+ },
+ "ranger.usersync.group.memberattributename": {
+ "value": ""
+ },
+ "ranger.usersync.pagedresultsenabled": {
+ "value": "true"
+ },
+ "ranger.usersync.pagedresultssize": {
+ "value": "500"
+ },
+ "ranger.usersync.source.impl.class": {
+ "value": "{USERSYNC_SYNC_SOURCE}",
+ "template": "yes"
+ },
+ "ranger.usersync.policymanager.baseURL": {
+ "value": "{{ranger_external_url}}"
+ },
+ "ranger.usersync.sleeptimeinmillisbetweensynccycle": {
+ "value": "60000"
+ },
+ "ranger.usersync.ldap.referral": {
+ "value": "ignore"
+ },
+ "ranger.usersync.unix.group.file": {
+ "value": "/etc/group"
+ },
+ "ranger.usersync.unix.password.file": {
+ "value": "/etc/passwd"
+ }
+ },
+ "usersync-properties":{
+ "SYNC_INTERVAL": {"remove": "yes"}
+ },
+ "webhcat-site":{
+ "templeton.hive.extra.files": "/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"
+ }
+ },
+ "property-mapping": {
+ "hive.heapsize": {
+ "map-to": "hive.heapsize",
+ "from-catalog": "hive-site",
+ "to-catalog": "hive-env",
+ "required-services": ["HIVE"]
+ },
+ "templeton.jar": {
+ "from-catalog": "webhcat-site",
+ "to-catalog": "webhcat-site",
+ "replace-from": "/usr/hdp/current/hive-webhcat",
+ "replace-to": "/usr/hdp/${hdp.version}/hive",
+ "required-services": [
+ "HIVE"
+ ]
+ },
+ "templeton.libjars": {
+ "from-catalog": "webhcat-site",
+ "to-catalog": "webhcat-site",
+ "replace-from": "/usr/hdp/current/zookeeper-client",
+ "replace-to": "/usr/hdp/${hdp.version}/zookeeper,/usr/hdp/${hdp.version}/hive/lib/hive-common.jar",
+ "required-services": [
+ "HIVE"
+ ]
+ },
+ "templeton.hadoop": {
+ "from-catalog": "webhcat-site",
+ "to-catalog": "webhcat-site",
+ "replace-from": "/usr/hdp/current/hadoop-client",
+ "replace-to": "/usr/hdp/${hdp.version}/hadoop",
+ "required-services": [
+ "HIVE"
+ ]
+ },
+ "templeton.hcat": {
+ "from-catalog": "webhcat-site",
+ "to-catalog": "webhcat-site",
+ "replace-from": "/usr/hdp/current/hive-client",
+ "replace-to": "/usr/hdp/${hdp.version}/hive",
+ "required-services": [
+ "HIVE"
+ ]
+ },
+ "https.attrib.clientAuth": {
+ "map-to": "ranger.service.https.attrib.clientAuth",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "https.attrib.keystoreFile": {
+ "map-to": "ranger.https.attrib.keystore.file",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "https.attrib.keystorePass": {
+ "map-to": "ranger.service.https.attrib.keystore.pass",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "https.service.port": {
+ "map-to": "ranger.service.https.port",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "http.service.port": {
+ "map-to": "ranger.service.http.port",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "https.attrib.keyAlias": {
+ "map-to": "ranger.service.https.attrib.keystore.keyalias",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "http.enabled": {
+ "map-to": "ranger.service.http.enabled",
+ "from-catalog": "ranger-site",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "authServiceHostName": {
+ "map-to": "ranger.unixauth.service.hostname",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "authServicePort": {
+ "map-to": "ranger.unixauth.service.port",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "authentication_method": {
+ "map-to": "ranger.authentication.method",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "remoteLoginEnabled": {
+ "map-to": "ranger.unixauth.remote.login.enabled",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_url": {
+ "map-to": "ranger.ldap.url",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_userDNpattern": {
+ "map-to": "ranger.ldap.user.dnpattern",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_groupSearchBase": {
+ "map-to": "ranger.ldap.group.searchbase",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_groupSearchFilter": {
+ "map-to": "ranger.ldap.group.searchfilter",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_groupRoleAttribute": {
+ "map-to": "ranger.ldap.group.roleattribute",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_ad_domain": {
+ "map-to": "ranger.ldap.ad.domain",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "xa_ldap_ad_url": {
+ "map-to": "ranger.ldap.ad.url",
+ "from-catalog": "admin-properties",
+ "to-catalog": "ranger-admin-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "CRED_KEYSTORE_FILENAME": {
+ "map-to": "ranger.usersync.credstore.filename",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "MIN_UNIX_USER_ID_TO_SYNC": {
+ "map-to": "ranger.usersync.unix.minUserId",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_BIND_DN": {
+ "map-to": "ranger.usersync.ldap.binddn",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_BIND_PASSWORD": {
+ "map-to": "ranger.usersync.ldap.ldapbindpassword",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_GROUPNAME_CASE_CONVERSION": {
+ "map-to": "ranger.usersync.ldap.groupname.caseconversion",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_URL": {
+ "map-to": "ranger.usersync.ldap.url",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USERNAME_CASE_CONVERSION": {
+ "map-to": "ranger.usersync.ldap.username.caseconversion",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USER_GROUP_NAME_ATTRIBUTE": {
+ "map-to": "ranger.usersync.ldap.user.groupnameattribute",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USER_NAME_ATTRIBUTE": {
+ "map-to": "ranger.usersync.ldap.user.nameattribute",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USER_OBJECT_CLASS": {
+ "map-to": "ranger.usersync.ldap.user.objectclass",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USER_SEARCH_BASE": {
+ "map-to": "ranger.usersync.ldap.user.searchbase",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USER_SEARCH_FILTER": {
+ "map-to": "ranger.usersync.ldap.user.searchfilter",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_LDAP_USER_SEARCH_SCOPE": {
+ "map-to": "ranger.usersync.ldap.user.searchscope",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "logdir": {
+ "map-to": "ranger.usersync.logdir",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "SYNC_SOURCE": {
+ "map-to": "ranger.usersync.sync.source",
+ "from-catalog": "usersync-properties",
+ "to-catalog": "ranger-ugsync-site",
+ "required-services": [
+ "RANGER"
+ ]
+ },
+ "nimbus.host": {
+ "map-to": "nimbus.seeds",
+ "coerce-to": "yaml-array"
+ },
+ "mapreduce.job.speculative.speculativecap": {
+ "map-to": "mapreduce.job.speculative.speculative-cap-running-tasks",
+ "from-catalog": "mapred-site",
+ "to-catalog": "mapred-site",
+ "default": "0.1"
+ },
+ "SSL_KEYSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.keystore",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-policymgr-ssl",
+ "default": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "SSL_KEYSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.keystore.password",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-policymgr-ssl",
+ "default": "myKeyFilePassword",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "SSL_TRUSTSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.truststore",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-policymgr-ssl",
+ "default": "/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "SSL_TRUSTSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.truststore.password",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-policymgr-ssl",
+ "default": "changeit",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "POLICY_MGR_URL": {
+ "map-to": "ranger.plugin.hdfs.policy.rest.url",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-security",
+ "default": "{POLICYMGR_MGR_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "REPOSITORY_NAME": {
+ "map-to": "ranger.plugin.hdfs.service.name",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-security",
+ "default": "{HDFS_RANGER_REPO_NAME}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+ "map-to": "xasecure.audit.destination.hdfs.dir",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-audit",
+ "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "XAAUDIT.DB.USER_NAME": {
+ "map-to": "xasecure.audit.destination.db.user",
+ "from-catalog": "ranger-hdfs-plugin-properties",
+ "to-catalog": "ranger-hdfs-audit",
+ "required-services": [
+ "RANGER",
+ "HDFS"
+ ]
+ },
+ "hbase_SSL_KEYSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.keystore",
+ "map-from": "SSL_KEYSTORE_FILE_PATH",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-policymgr-ssl",
+ "default": "/usr/hdp/current/hbase-client/conf/ranger-plugin-keystore.jks",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_SSL_KEYSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.keystore.password",
+ "map-from": "SSL_KEYSTORE_PASSWORD",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-policymgr-ssl",
+ "default": "myKeyFilePassword",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_SSL_TRUSTSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.truststore",
+ "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-policymgr-ssl",
+ "default": "/usr/hdp/current/hbase-client/conf/ranger-plugin-truststore.jks",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_SSL_TRUSTSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.truststore.password",
+ "map-from": "SSL_TRUSTSTORE_PASSWORD",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-policymgr-ssl",
+ "default": "changeit",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": {
+ "map-to": "xasecure.hbase.update.xapolicies.on.grant.revoke",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-security",
+ "default": "true",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_POLICY_MGR_URL": {
+ "map-from": "POLICY_MGR_URL",
+ "map-to": "ranger.plugin.hbase.policy.rest.url",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-security",
+ "default": "{POLICYMGR_MGR_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_REPOSITORY_NAME": {
+ "map-from": "REPOSITORY_NAME",
+ "map-to": "ranger.plugin.hbase.service.name",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-security",
+ "default": "{HBASE_RANGER_REPO_NAME}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+ "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+ "map-to": "xasecure.audit.destination.hdfs.dir",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-audit",
+ "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hbase_XAAUDIT.DB.USER_NAME": {
+ "map-from": "XAAUDIT.DB.USER_NAME",
+ "map-to": "xasecure.audit.destination.db.user",
+ "from-catalog": "ranger-hbase-plugin-properties",
+ "to-catalog": "ranger-hbase-audit",
+ "required-services": [
+ "RANGER",
+ "HBASE"
+ ]
+ },
+ "hive_SSL_KEYSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.keystore",
+ "map-from": "SSL_KEYSTORE_FILE_PATH",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-policymgr-ssl",
+ "default": "/etc/hive/conf/conf.server/ranger-plugin-keystore.jks",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_SSL_KEYSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.keystore.password",
+ "map-from": "SSL_KEYSTORE_PASSWORD",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-policymgr-ssl",
+ "default": "myKeyFilePassword",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_SSL_TRUSTSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.truststore",
+ "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-policymgr-ssl",
+ "default": "/etc/hive/conf/conf.server/ranger-plugin-truststore.jks",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_SSL_TRUSTSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.truststore.password",
+ "map-from": "SSL_TRUSTSTORE_PASSWORD",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-policymgr-ssl",
+ "default": "changeit",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_UPDATE_XAPOLICIES_ON_GRANT_REVOKE": {
+ "map-from": "UPDATE_XAPOLICIES_ON_GRANT_REVOKE",
+ "map-to": "xasecure.hive.update.xapolicies.on.grant.revoke",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-security",
+ "default": "true",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_POLICY_MGR_URL": {
+ "map-from": "POLICY_MGR_URL",
+ "map-to": "ranger.plugin.hive.policy.rest.url",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-security",
+ "default": "{POLICYMGR_MGR_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_REPOSITORY_NAME": {
+ "map-from": "REPOSITORY_NAME",
+ "map-to": "ranger.plugin.hive.service.name",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-security",
+ "default": "{HIVE_RANGER_REPO_NAME}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+ "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+ "map-to": "xasecure.audit.destination.hdfs.dir",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-audit",
+ "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "hive_XAAUDIT.DB.USER_NAME": {
+ "map-from": "XAAUDIT.DB.USER_NAME",
+ "map-to": "xasecure.audit.destination.db.user",
+ "from-catalog": "ranger-hive-plugin-properties",
+ "to-catalog": "ranger-hive-audit",
+ "required-services": [
+ "RANGER",
+ "HIVE"
+ ]
+ },
+ "knox_SSL_KEYSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.keystore",
+ "map-from": "SSL_KEYSTORE_FILE_PATH",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-policymgr-ssl",
+ "default": "/usr/hdp/current/knox-server/conf/ranger-plugin-keystore.jks",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_SSL_KEYSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.keystore.password",
+ "map-from": "SSL_KEYSTORE_PASSWORD",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-policymgr-ssl",
+ "default": "myKeyFilePassword",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_SSL_TRUSTSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.truststore",
+ "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-policymgr-ssl",
+ "default": "/usr/hdp/current/knox-server/conf/ranger-plugin-truststore.jks",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_SSL_TRUSTSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.truststore.password",
+ "map-from": "SSL_TRUSTSTORE_PASSWORD",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-policymgr-ssl",
+ "default": "changeit",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_POLICY_MGR_URL": {
+ "map-from": "POLICY_MGR_URL",
+ "map-to": "ranger.plugin.knox.policy.rest.url",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-security",
+ "default": "{POLICYMGR_MGR_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_REPOSITORY_NAME": {
+ "map-from": "REPOSITORY_NAME",
+ "map-to": "ranger.plugin.knox.service.name",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-security",
+ "default": "{KNOX_RANGER_REPO_NAME}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+ "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+ "map-to": "xasecure.audit.destination.hdfs.dir",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-audit",
+ "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "knox_XAAUDIT.DB.USER_NAME": {
+ "map-from": "XAAUDIT.DB.USER_NAME",
+ "map-to": "xasecure.audit.destination.db.user",
+ "from-catalog": "ranger-knox-plugin-properties",
+ "to-catalog": "ranger-knox-audit",
+ "required-services": [
+ "RANGER",
+ "KNOX"
+ ]
+ },
+ "storm_SSL_KEYSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.keystore",
+ "map-from": "SSL_KEYSTORE_FILE_PATH",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-policymgr-ssl",
+ "default": "/usr/hdp/current/storm-client/conf/ranger-plugin-keystore.jks",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_SSL_KEYSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.keystore.password",
+ "map-from": "SSL_KEYSTORE_PASSWORD",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-policymgr-ssl",
+ "default": "myKeyFilePassword",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_SSL_TRUSTSTORE_FILE_PATH": {
+ "map-to": "xasecure.policymgr.clientssl.truststore",
+ "map-from": "SSL_TRUSTSTORE_FILE_PATH",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-policymgr-ssl",
+ "default": "/usr/hdp/current/storm-client/conf/ranger-plugin-truststore.jks",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_SSL_TRUSTSTORE_PASSWORD": {
+ "map-to": "xasecure.policymgr.clientssl.truststore.password",
+ "map-from": "SSL_TRUSTSTORE_PASSWORD",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-policymgr-ssl",
+ "default": "changeit",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_POLICY_MGR_URL": {
+ "map-from": "POLICY_MGR_URL",
+ "map-to": "ranger.plugin.storm.policy.rest.url",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-security",
+ "default": "{POLICYMGR_MGR_URL}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_REPOSITORY_NAME": {
+ "map-from": "REPOSITORY_NAME",
+ "map-to": "ranger.plugin.storm.service.name",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-security",
+ "default": "{STORM_RANGER_REPO_NAME}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_XAAUDIT.HDFS.DESTINATION_DIRECTORY": {
+ "map-from": "XAAUDIT.HDFS.DESTINATION_DIRECTORY",
+ "map-to": "xasecure.audit.destination.hdfs.dir",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-audit",
+ "default": "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}",
+ "template": "yes",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ },
+ "storm_XAAUDIT.DB.USER_NAME": {
+ "map-from": "XAAUDIT.DB.USER_NAME",
+ "map-to": "xasecure.audit.destination.db.user",
+ "from-catalog": "ranger-storm-plugin-properties",
+ "to-catalog": "ranger-storm-audit",
+ "required-services": [
+ "RANGER",
+ "STORM"
+ ]
+ }
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3_step2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3_step2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3_step2.json
new file mode 100644
index 0000000..014aa71
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3_step2.json
@@ -0,0 +1,81 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.2",
+ "target-version": "2.3",
+ "options": {
+ "config-types": {
+ "ranger-hdfs-security": {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","HDFS"]
+ },
+ "ranger-hbase-security": {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","HBASE"]
+ },
+ "ranger-hive-security": {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","HIVE"]
+ },
+ "ranger-knox-security": {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","KNOX"]
+ },
+ "ranger-storm-security": {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","STORM"]
+ },
+ "ranger-kafka-security" : {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","KAFKA"]
+ },
+ "ranger-yarn-security" : {
+ "merged-copy": "yes",
+ "required-services": ["RANGER","YARN"]
+ },
+ "ranger-admin-site": {
+ "merged-copy": "yes",
+ "required-services": [ "RANGER" ]
+ },
+ "ranger-ugsync-site": {
+ "merged-copy": "yes",
+ "required-services": [
+ "RANGER"
+ ]
+ }
+ }
+ },
+ "properties": {
+ "ranger-admin-site": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-ugsync-site": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-hbase-security": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-hdfs-security": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-storm-security": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-knox-security": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-hive-security": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-kafka-security": {
+ "upgrade.transition": {"remove": "yes"}
+ },
+ "ranger-yarn-security": {
+ "upgrade.transition": {"remove": "yes"}
+ }
+ }
+ }
+ ]
+}
[30/55] [abbrv] ambari git commit: AMBARI-18165. (Additional
fix).Alert on Atlas after adding it to a secure cluster as HBase table
initialization fails.(vbrodetskyi)
Posted by ol...@apache.org.
AMBARI-18165. (Additional fix).Alert on Atlas after adding it to a secure cluster as HBase table initialization fails.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da07ea14
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da07ea14
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da07ea14
Branch: refs/heads/branch-dev-logsearch
Commit: da07ea14a911837e709f330b33b205cb17f80f05
Parents: ae47ae9
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Sat Aug 20 00:12:05 2016 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Sat Aug 20 00:13:00 2016 +0300
----------------------------------------------------------------------
.../ATLAS/0.1.0.2.3/package/scripts/metadata_server.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/da07ea14/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index d585c77..85ecb15 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -19,7 +19,8 @@ limitations under the License.
from metadata import metadata
from resource_management.libraries.functions import conf_select, stack_select
from resource_management.core.resources.system import Execute, File
-from resource_management.libraries.script.script import Script, format_stack_version
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.check_process_status import check_process_status
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.security_commons import build_expectations, \
[40/55] [abbrv] ambari git commit: AMBARI-18208. Bug Fixing in
HueMIgration View. (Pradarttana Panda via gauravn7)
Posted by ol...@apache.org.
AMBARI-18208. Bug Fixing in HueMIgration View. (Pradarttana Panda via gauravn7)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a34fe1d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a34fe1d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a34fe1d
Branch: refs/heads/branch-dev-logsearch
Commit: 7a34fe1d80897d5db7acb0ac276fb270b3baa3b9
Parents: 7494549
Author: Gaurav Nagar <gn...@hortonworks.com>
Authored: Sun Aug 21 20:43:55 2016 +0530
Committer: Gaurav Nagar <gn...@hortonworks.com>
Committed: Sun Aug 21 20:45:02 2016 +0530
----------------------------------------------------------------------
.../historyqueryset/MysqlQuerySetAmbariDB.java | 20 ++-
.../historyqueryset/OracleQuerySetAmbariDB.java | 19 ++-
.../hive/historyqueryset/QuerySetAmbariDB.java | 68 ++++++++--
.../hive/instancedetail/QuerySetAmbariDB.java | 4 +-
.../savedqueryset/MysqlQuerySetAmbariDB.java | 16 ++-
.../savedqueryset/OracleQuerySetAmbariDB.java | 12 +-
.../hive/savedqueryset/QuerySetAmbariDB.java | 37 ++++--
.../pig/instancedetail/QuerySetAmbariDB.java | 2 +-
.../pig/jobqueryset/MysqlQuerySetAmbariDB.java | 10 +-
.../pig/jobqueryset/OracleQuerySetAmbariDB.java | 10 +-
.../pig/jobqueryset/QuerySetAmbariDB.java | 37 ++++--
.../MysqlQuerySetAmbariDB.java | 10 +-
.../OracleQuerySetAmbariDB.java | 18 ++-
.../savedscriptqueryset/QuerySetAmbariDB.java | 31 ++++-
.../HiveHistoryMigrationUtility.java | 32 +++--
...HiveHistoryQueryMigrationImplementation.java | 94 +++++++++++---
.../HiveSavedQueryMigrationImplementation.java | 82 ++++++++----
.../HiveSavedQueryMigrationUtility.java | 103 ++++++---------
.../hive/savedquery/HiveSavedQueryStartJob.java | 2 -
.../pigjob/PigJobMigrationImplementation.java | 83 ++++++++----
.../pig/pigjob/PigJobMigrationUtility.java | 106 +++++++---------
.../pig/pigscript/PigSavedScriptStartJob.java | 2 -
.../PigScriptMigrationImplementation.java | 125 +++++++++++++++++--
.../pigscript/PigScriptMigrationUtility.java | 32 +++--
.../scripts/models/MigrationModel.java | 18 +--
.../app/models/checkprogress.js | 2 +-
.../app/models/returnjobid.js | 2 +-
.../app/models/startmigration.js | 2 +-
.../app/routes/home-page/hive-history.js | 11 +-
.../app/routes/home-page/hive-saved-query.js | 8 +-
.../app/routes/home-page/pig-job.js | 8 +-
.../app/routes/home-page/pig-script.js | 8 +-
.../app/routes/home-page/revert-change.js | 10 +-
.../src/main/resources/view.xml | 9 +-
34 files changed, 735 insertions(+), 298 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/MysqlQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/MysqlQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/MysqlQuerySetAmbariDB.java
index c08f009..4c6bffa 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/MysqlQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/MysqlQuerySetAmbariDB.java
@@ -30,12 +30,15 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
@Override
protected String getTableIdSqlFromInstanceName() {
- return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
+ return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.%hive%.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
+ }
+ @Override
+ protected String getSqlInsertHiveHistoryForHive(int id) {
+ return "INSERT INTO DS_JOBIMPL_" + id + " values (?,'','','','','default',?,0,'','','',?,?,?,'','job','','','UNKNOWN',?,'','Worksheet');";
}
-
@Override
- protected String getSqlInsertHiveHistory(int id) {
- return "INSERT INTO DS_JOBIMPL_" + id + " values (?,'','','','','default',?,0,'','',?,'admin',?,'','job','','','Unknown',?,'','Worksheet');";
+ protected String getSqlInsertHiveHistoryForHiveNext(int id) {
+ return "INSERT INTO DS_JOBIMPL_" + id + " values (?,'','','','','default',?,0,'','','','',?,?,?,'','job','','','UNKNOWN',?,'','Worksheet');";
}
@Override
@@ -43,4 +46,13 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
return "delete from DS_JOBIMPL_" + id + " where ds_id='" + maxcount + "';";
}
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences ambari_sequences set sequence_value=? where sequence_name='ds_jobimpl_"+id+"_id_seq';";
+ }
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_jobimpl_"+id+"_id_seq';";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/OracleQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/OracleQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/OracleQuerySetAmbariDB.java
index f3349c6..7a197d3 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/OracleQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/OracleQuerySetAmbariDB.java
@@ -32,13 +32,28 @@ public class OracleQuerySetAmbariDB extends QuerySetAmbariDB {
protected String getTableIdSqlFromInstanceName() {
return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl' and view_instance_name=?";
}
+
@Override
- protected String getSqlInsertHiveHistory(int id) {
- return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','',?,'admin',?,'','job','','','Unknown',?,'','Worksheet')";
+ protected String getSqlInsertHiveHistoryForHive(int id) {
+ return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','','',?,?,?,'','job','','','UNKNOWN',?,'','Worksheet')";
}
@Override
+ protected String getSqlInsertHiveHistoryForHiveNext(int id) {
+ return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','','','',?,?,?,'','job','','','UNKNOWN',?,'','Worksheet')";
+ }
+
+ @Override
protected String getRevSql(int id,String maxcount){
return "delete from ds_jobimpl_" + id + " where ds_id='" + maxcount + "'";
}
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_jobimpl_"+id+"_id_seq'";
+ }
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_jobimpl_"+id+"_id_seq'";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/QuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/QuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/QuerySetAmbariDB.java
index 5be8cc1..dbf4f26 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/QuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/historyqueryset/QuerySetAmbariDB.java
@@ -33,6 +33,20 @@ public abstract class QuerySetAmbariDB {
return prSt;
}
+ public PreparedStatement getSequenceNoFromAmbariSequence(Connection connection,int id) throws SQLException {
+ PreparedStatement prSt = connection.prepareStatement(getSqlSequenceNoFromAmbariSequence(id));
+ return prSt;
+ }
+
+ public PreparedStatement updateSequenceNoInAmbariSequence(Connection connection, int seqNo, int id) throws SQLException {
+
+ PreparedStatement prSt = connection.prepareStatement(getSqlUpdateSequenceNo(id));
+
+ prSt.setInt(1, seqNo);
+
+ return prSt;
+ }
+
public PreparedStatement getMaxDsIdFromTableId(Connection connection, int id) throws SQLException {
PreparedStatement prSt = connection.prepareStatement(getSqlMaxDSidFromTableId(id));
@@ -40,23 +54,50 @@ public abstract class QuerySetAmbariDB {
return prSt;
}
- public PreparedStatement insertToHiveHistory(Connection connection, int id, String maxcount, long epochtime, String dirname) throws SQLException {
+ public PreparedStatement insertToHiveHistoryForHive(Connection connection, int id, String maxcount, long epochtime, String dirname,String username) throws SQLException {
+
+ String Logfile= dirname + "logs";
+ String queryHqlFile= dirname + "query.hql";
+
+ PreparedStatement prSt = connection.prepareStatement(getSqlInsertHiveHistoryForHive(id));
+
+ prSt.setString(1, maxcount);
+ prSt.setLong(2, epochtime);
+ prSt.setString(3, Logfile);
+ prSt.setString(4, username);
+ prSt.setString(5, queryHqlFile);
+ prSt.setString(6, dirname);
+
+ return prSt;
+ }
+
+ public PreparedStatement insertToHiveHistoryForHiveNext(Connection connection, int id, String maxcount, long epochtime, String dirname,String username) throws SQLException {
String Logfile= dirname + "logs";
String queryHqlFile= dirname + "query.hql";
- PreparedStatement prSt = connection.prepareStatement(getSqlInsertHiveHistory(id));
+ PreparedStatement prSt = connection.prepareStatement(getSqlInsertHiveHistoryForHiveNext(id));
prSt.setString(1, maxcount);
prSt.setLong(2, epochtime);
prSt.setString(3, Logfile);
- prSt.setString(4, queryHqlFile);
- prSt.setString(5, dirname);
+ prSt.setString(4, username);
+ prSt.setString(5, queryHqlFile);
+ prSt.setString(6, dirname);
return prSt;
}
- public String RevertSql(int id,String maxcount) throws SQLException {
+ public PreparedStatement getHiveVersionInstance(Connection connection,String viewName) throws SQLException {
+ PreparedStatement prSt = connection.prepareStatement(getHiveVersionDetailSql());
+ prSt.setString(1, viewName);
+ return prSt;
+ }
+
+ protected String getHiveVersionDetailSql(){
+ return "select distinct(view_name) as viewname from viewentity where view_instance_name =?;";
+ }
+ public String revertSql(int id, String maxcount) throws SQLException {
return getRevSql(id,maxcount);
}
@@ -65,15 +106,26 @@ public abstract class QuerySetAmbariDB {
}
protected String getTableIdSqlFromInstanceName() {
- return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
+ return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.%hive%.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
+ }
+
+ protected String getSqlInsertHiveHistoryForHive(int id) {
+ return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','','',?,?,?,'','job','','','UNKNOWN',?,'','Worksheet');";
}
- protected String getSqlInsertHiveHistory(int id) {
- return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','',?,'admin',?,'','job','','','Unknown',?,'','Worksheet');";
+ protected String getSqlInsertHiveHistoryForHiveNext(int id) {
+ return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','','','',?,?,?,'','job','','','UNKNOWN',?,'','Worksheet');";
}
protected String getRevSql(int id,String maxcount){
return "delete from ds_jobimpl_" + id + " where ds_id='" + maxcount + "';";
}
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_jobimpl_"+id+"_id_seq';";
+ }
+
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_jobimpl_"+id+"_id_seq';";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/instancedetail/QuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/instancedetail/QuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/instancedetail/QuerySetAmbariDB.java
index da1c411..21bd32e 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/instancedetail/QuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/instancedetail/QuerySetAmbariDB.java
@@ -38,11 +38,11 @@ public abstract class QuerySetAmbariDB {
}
protected String getHiveInstanceSql(){
- return "select distinct(view_instance_name) as instancename from viewentity where view_name='HIVE{1.0.0}';";
+ return "select distinct(view_instance_name) as instancename from viewentity where view_name like '%HIVE%';";
}
protected String getAllInstanceDetailSql(){
- return "select distinct(view_instance_name) as instancename from viewentity where view_name='HIVE{1.0.0}' or view_name='PIG{1.0.0}';";
+ return "select distinct(view_instance_name) as instancename from viewentity where view_name like '%HIVE%' or view_name like '%PIG%';";
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/MysqlQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/MysqlQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/MysqlQuerySetAmbariDB.java
index 2b40491..204ace9 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/MysqlQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/MysqlQuerySetAmbariDB.java
@@ -30,7 +30,7 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
@Override
protected String getTableIdSqlFromInstanceNameSavedQuery() {
- return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.savedQueries.SavedQuery' and view_instance_name=?;";
+ return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.%hive%.resources.savedQueries.SavedQuery' and view_instance_name=?;";
}
@Override
@@ -40,17 +40,17 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
@Override
protected String getTableIdSqlFromInstanceNameHistoryQuery() {
- return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
+ return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.%hive%.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
}
@Override
protected String getSqlInsertHiveHistory(int id) {
- return "INSERT INTO DS_JOBIMPL_" + id + " values (?,'','','','','default',?,0,'','',?,'admin',?,'','job','','','Unknown',?,'','Worksheet');";
+ return "INSERT INTO DS_JOBIMPL_" + id + " values (?,'','','','','default',?,0,'','',?,?,?,'','job','','','Unknown',?,'','Worksheet');";
}
@Override
protected String getSqlInsertSavedQuery(int id) {
- return "INSERT INTO DS_SAVEDQUERY_" + id + " values (?,?,'" + "admin" + "',?,?,?);";
+ return "INSERT INTO DS_SAVEDQUERY_" + id + " values (?,?,?,?,?,?);";
}
@Override
@@ -62,4 +62,12 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
protected String getRevSqlHistoryQuery(int id, String maxcount) {
return "delete from DS_JOBIMPL_" + id + " where ds_id='" + maxcount + "';";
}
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_savedquery_"+id+"_id_seq';";
+ }
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_savedquery_"+id+"_id_seq';";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/OracleQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/OracleQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/OracleQuerySetAmbariDB.java
index 11f9170..5efcc1c 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/OracleQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/OracleQuerySetAmbariDB.java
@@ -40,11 +40,11 @@ public class OracleQuerySetAmbariDB extends QuerySetAmbariDB {
@Override
protected String getSqlInsertHiveHistory(int id) {
- return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','',?,'admin',?,'','job','','','Unknown',?,'','Worksheet')";
+ return "INSERT INTO ds_jobimpl_" + id + " values (?,'','','','','default',?,0,'','',?,?,?,'','job','','','Unknown',?,'','Worksheet')";
}
@Override
protected String getSqlInsertSavedQuery(int id) {
- return "INSERT INTO ds_savedquery_" + id + " values (?,?,'" + "admin" + "',?,?,?)";
+ return "INSERT INTO ds_savedquery_" + id + " values (?,?,?,?,?,?)";
}
@Override
@@ -55,4 +55,12 @@ public class OracleQuerySetAmbariDB extends QuerySetAmbariDB {
protected String getRevSqlHistoryQuery(int id, String maxcount) {
return "delete from ds_jobimpl_" + id + " where ds_id='" + maxcount + "'";
}
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_savedquery_"+id+"_id_seq'";
+ }
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_savedquery_"+id+"_id_seq'";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/QuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/QuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/QuerySetAmbariDB.java
index 8ab13b2..3bfe73d 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/QuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/hive/savedqueryset/QuerySetAmbariDB.java
@@ -34,6 +34,20 @@ public abstract class QuerySetAmbariDB {
return prSt;
}
+ public PreparedStatement updateSequenceNoInAmbariSequence(Connection connection, int seqNo, int id) throws SQLException {
+
+ PreparedStatement prSt = connection.prepareStatement(getSqlUpdateSequenceNo(id));
+
+ prSt.setInt(1, seqNo);
+
+ return prSt;
+ }
+
+ public PreparedStatement getSequenceNoFromAmbariSequence(Connection connection,int id) throws SQLException {
+ PreparedStatement prSt = connection.prepareStatement(getSqlSequenceNoFromAmbariSequence(id));
+ return prSt;
+ }
+
public PreparedStatement getTableIdFromInstanceNameHistoryquery(Connection connection, String instance) throws SQLException {
PreparedStatement prSt = connection.prepareStatement(getTableIdSqlFromInstanceNameHistoryQuery());
@@ -70,7 +84,7 @@ public abstract class QuerySetAmbariDB {
return prSt;
}
- public PreparedStatement insertToHiveSavedQuery(Connection connection, int id, String maxcount, String database, String dirname, String query, String name) throws SQLException {
+ public PreparedStatement insertToHiveSavedQuery(Connection connection, int id, String maxcount, String database, String dirname, String query, String name,String username) throws SQLException {
String Logfile = dirname + "logs";
String queryHqlFile = dirname + "query.hql";
@@ -79,9 +93,10 @@ public abstract class QuerySetAmbariDB {
prSt.setString(1, maxcount);
prSt.setString(2, database);
- prSt.setString(3, queryHqlFile);
- prSt.setString(4, query);
- prSt.setString(5, name);
+ prSt.setString(3, username);
+ prSt.setString(4, queryHqlFile);
+ prSt.setString(5, query);
+ prSt.setString(6, name);
return prSt;
}
@@ -101,7 +116,7 @@ public abstract class QuerySetAmbariDB {
}
protected String getTableIdSqlFromInstanceNameSavedQuery() {
- return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.savedQueries.SavedQuery' and view_instance_name=?;";
+ return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.%hive%.resources.savedQueries.SavedQuery' and view_instance_name=?;";
}
protected String getSqlMaxDSidFromTableIdHistoryQuery(int id) {
@@ -109,7 +124,7 @@ public abstract class QuerySetAmbariDB {
}
protected String getTableIdSqlFromInstanceNameHistoryQuery() {
- return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
+ return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.%hive%.resources.jobs.viewJobs.JobImpl' and view_instance_name=?;";
}
protected String getSqlInsertHiveHistory(int id) {
@@ -117,7 +132,7 @@ public abstract class QuerySetAmbariDB {
}
protected String getSqlInsertSavedQuery(int id) {
- return "INSERT INTO ds_savedquery_" + id + " values (?,?,'" + "admin" + "',?,?,?);";
+ return "INSERT INTO ds_savedquery_" + id + " values (?,?,?,?,?,?);";
}
protected String getRevSqlSavedQuery(int id, String maxcount) {
@@ -128,4 +143,12 @@ public abstract class QuerySetAmbariDB {
return "delete from ds_jobimpl_" + id + " where ds_id='" + maxcount + "';";
}
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_savedquery_"+id+"_id_seq';";
+ }
+
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_savedquery_"+id+"_id_seq';";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/instancedetail/QuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/instancedetail/QuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/instancedetail/QuerySetAmbariDB.java
index 6d878fe..1241811 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/instancedetail/QuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/instancedetail/QuerySetAmbariDB.java
@@ -33,7 +33,7 @@ public abstract class QuerySetAmbariDB {
}
protected String getPigInstanceSql(){
- return "select distinct(view_instance_name) as instancename from viewentity where view_name='PIG{1.0.0}';";
+ return "select distinct(view_instance_name) as instancename from viewentity where view_name like '%PIG%';";
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/MysqlQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/MysqlQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/MysqlQuerySetAmbariDB.java
index fdc2484..b85a552 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/MysqlQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/MysqlQuerySetAmbariDB.java
@@ -34,10 +34,18 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
}
@Override
protected String getSqlinsertToPigJob(int id) {
- return "INSERT INTO DS_PIGJOB_" + id + " values (?,?,0,'','f','','','admin',0,?,'',?,'','',?,?,'',?);";
+ return "INSERT INTO DS_PIGJOB_" + id + " values (?,?,0,'',0,'','',?,0,?,'',?,'','',?,?,'',?);";
}
@Override
protected String getRevSql(int id, String maxcount) {
return "delete from DS_PIGJOB_" + id + " where ds_id='" + maxcount + "';";
}
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_pigjob_"+id+"_id_seq';";
+ }
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_pigjob_"+id+"_id_seq';";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/OracleQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/OracleQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/OracleQuerySetAmbariDB.java
index 2c27409..dea8429 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/OracleQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/OracleQuerySetAmbariDB.java
@@ -32,10 +32,18 @@ public class OracleQuerySetAmbariDB extends QuerySetAmbariDB {
}
@Override
protected String getSqlinsertToPigJob(int id) {
- return "INSERT INTO ds_pigjob_" + id + " values (?,?,0,'','f','','','admin',0,?,'',?,'','',?,?,'',?)";
+ return "INSERT INTO ds_pigjob_" + id + " values (?,?,0,'','f','','',?,0,?,'',?,'','',?,?,'',?)";
}
@Override
protected String getRevSql(int id, String maxcount) {
return "delete from ds_pigjob_" + id + " where ds_id='" + maxcount + "'";
}
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_pigjob_"+id+"_id_seq'";
+ }
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_pigjob_"+id+"_id_seq'";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/QuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/QuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/QuerySetAmbariDB.java
index ec70e45..7dddd18 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/QuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/jobqueryset/QuerySetAmbariDB.java
@@ -34,13 +34,27 @@ public abstract class QuerySetAmbariDB {
return prSt;
}
+ public PreparedStatement getSequenceNoFromAmbariSequence(Connection connection,int id) throws SQLException {
+ PreparedStatement prSt = connection.prepareStatement(getSqlSequenceNoFromAmbariSequence(id));
+ return prSt;
+ }
+
public PreparedStatement getMaxDsIdFromTableId(Connection connection, int id) throws SQLException {
PreparedStatement prSt = connection.prepareStatement(getSqlMaxDSidFromTableId(id));
return prSt;
}
- public PreparedStatement insertToPigJob(String dirname, String maxcountforpigjob, long epochtime1, String title, Connection connection, int id, String status) throws SQLException {
+ public PreparedStatement updateSequenceNoInAmbariSequence(Connection connection, int seqNo, int id) throws SQLException {
+
+ PreparedStatement prSt = connection.prepareStatement(getSqlUpdateSequenceNo(id));
+
+ prSt.setInt(1, seqNo);
+
+ return prSt;
+ }
+
+ public PreparedStatement insertToPigJob(String dirname, String maxcountforpigjob, long epochtime1, String title, Connection connection, int id, String status,String username) throws SQLException {
String pigScriptFile = dirname + "script.pig";
@@ -48,11 +62,12 @@ public abstract class QuerySetAmbariDB {
prSt.setString(1, maxcountforpigjob);
prSt.setLong(2, epochtime1);
- prSt.setString(3, pigScriptFile);
- prSt.setString(4, maxcountforpigjob);
- prSt.setString(5, status);
- prSt.setString(6, dirname);
- prSt.setString(7, title);
+ prSt.setString(3, username);
+ prSt.setString(4, pigScriptFile);
+ prSt.setString(5, maxcountforpigjob);
+ prSt.setString(6, status);
+ prSt.setString(7, dirname);
+ prSt.setString(8, title);
return prSt;
}
@@ -70,11 +85,19 @@ public abstract class QuerySetAmbariDB {
}
protected String getSqlinsertToPigJob(int id) {
- return "INSERT INTO ds_pigjob_" + id + " values (?,?,0,'','f','','','admin',0,?,'',?,'','',?,?,'',?);";
+ return "INSERT INTO ds_pigjob_" + id + " values (?,?,0,'','f','','',?,0,?,'',?,'','',?,?,'',?);";
}
protected String getRevSql(int id, String maxcount) {
return "delete from ds_pigjob_" + id + " where ds_id='" + maxcount + "';";
}
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_pigjob_"+id+"_id_seq';";
+ }
+
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_pigjob_"+id+"_id_seq';";
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/MysqlQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/MysqlQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/MysqlQuerySetAmbariDB.java
index a8c1c4d..878059f 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/MysqlQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/MysqlQuerySetAmbariDB.java
@@ -33,11 +33,19 @@ public class MysqlQuerySetAmbariDB extends QuerySetAmbariDB {
}
@Override
protected String getSqlinsertToPigScript(int id) {
- return "INSERT INTO DS_PIGSCRIPT_" + id + " values (?,'1970-01-17 20:28:55.586000 +00:00:00',0,'admin',?,'','',?);";
+ return "INSERT INTO DS_PIGSCRIPT_" + id + " values (?,'1970-01-17 20:28:55.586000 +00:00:00',0,?,?,'','',?);";
}
@Override
protected String getRevSql(int id,String maxcount){
return "delete from DS_PIGSCRIPT_" + id + " where ds_id='" + maxcount + "';";
}
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_pigscript_"+id+"_id_seq';";
+ }
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_pigscript_"+id+"_id_seq';";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/OracleQuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/OracleQuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/OracleQuerySetAmbariDB.java
index d7dfc0b..101c819 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/OracleQuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/OracleQuerySetAmbariDB.java
@@ -21,21 +21,29 @@ public class OracleQuerySetAmbariDB extends QuerySetAmbariDB {
/*
* Overriding methods for Oracle specific queries
* */
-
+ @Override
protected String getSqlMaxDSidFromTableId(int id) {
return "select MAX(cast(ds_id as integer)) as max from ds_pigscript_" + id + "";
}
-
+ @Override
protected String getTableIdSqlFromInstanceName() {
return "select id from viewentity where class_name LIKE 'org.apache.ambari.view.pig.resources.scripts.models.PigScript' and view_instance_name=?";
}
-
+ @Override
protected String getSqlinsertToPigScript(int id) {
- return "INSERT INTO ds_pigscript_" + id + " values (?,'1970-01-17 20:28:55.586000 +00:00:00','f','admin',?,'','',?)";
+ return "INSERT INTO ds_pigscript_" + id + " values (?,'1970-01-17 20:28:55.586000 +00:00:00','f',?,?,'','',?)";
}
-
+ @Override
protected String getRevSql(int id, String maxcount) {
return "delete from ds_pigscript_" + id + " where ds_id='" + maxcount + "'";
}
+ @Override
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_pigscript_"+id+"_id_seq'";
+ }
+ @Override
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_pigscript_"+id+"_id_seq'";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/QuerySetAmbariDB.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/QuerySetAmbariDB.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/QuerySetAmbariDB.java
index 4fc61fe..d032264 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/QuerySetAmbariDB.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/datasource/queryset/ambariqueryset/pig/savedscriptqueryset/QuerySetAmbariDB.java
@@ -32,18 +32,24 @@ public abstract class QuerySetAmbariDB {
return prSt;
}
+ public PreparedStatement getSequenceNoFromAmbariSequence(Connection connection,int id) throws SQLException {
+ PreparedStatement prSt = connection.prepareStatement(getSqlSequenceNoFromAmbariSequence(id));
+ return prSt;
+ }
+
public PreparedStatement getMaxDsIdFromTableId(Connection connection, int id) throws SQLException {
PreparedStatement prSt = connection.prepareStatement(getSqlMaxDSidFromTableId(id));
return prSt;
}
- public PreparedStatement insertToPigScript(Connection connection, int id, String maxcount1, String dirname, String title) throws SQLException {
+ public PreparedStatement insertToPigScript(Connection connection, int id, String maxcount1, String dirname, String title,String username) throws SQLException {
PreparedStatement prSt = connection.prepareStatement(getSqlinsertToPigScript(id));
prSt.setString(1, maxcount1);
- prSt.setString(2, dirname);
- prSt.setString(3, title);
+ prSt.setString(2, username);
+ prSt.setString(3, dirname);
+ prSt.setString(4, title);
return prSt;
}
@@ -52,6 +58,15 @@ public abstract class QuerySetAmbariDB {
return getRevSql(id, maxcount);
}
+ public PreparedStatement updateSequenceNoInAmbariSequence(Connection connection, int seqNo, int id) throws SQLException {
+
+ PreparedStatement prSt = connection.prepareStatement(getSqlUpdateSequenceNo(id));
+
+ prSt.setInt(1, seqNo);
+
+ return prSt;
+ }
+
protected String getSqlMaxDSidFromTableId(int id) {
return "select MAX(cast(ds_id as integer)) as max from ds_pigscript_" + id + ";";
}
@@ -61,10 +76,18 @@ public abstract class QuerySetAmbariDB {
}
protected String getSqlinsertToPigScript(int id) {
- return "INSERT INTO ds_pigscript_" + id + " values (?,'1970-01-17 20:28:55.586000 +00:00:00','f','admin',?,'','',?);";
+ return "INSERT INTO ds_pigscript_" + id + " values (?,'1970-01-17 20:28:55.586000 +00:00:00','f',?,?,'','',?);";
}
protected String getRevSql(int id, String maxcount) {
return "delete from ds_pigscript_" + id + " where ds_id='" + maxcount + "';";
}
+
+ protected String getSqlSequenceNoFromAmbariSequence(int id) {
+ return "select sequence_value from ambari_sequences where sequence_name ='ds_pigscript_"+id+"_id_seq';";
+ }
+
+ protected String getSqlUpdateSequenceNo(int id) {
+ return "update ambari_sequences set sequence_value=? where sequence_name='ds_pigscript_"+id+"_id_seq';";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryMigrationUtility.java
index bdcf293..2c205b0 100755
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryMigrationUtility.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryMigrationUtility.java
@@ -79,7 +79,7 @@ public class HiveHistoryMigrationUtility {
MigrationModel model = new MigrationModel();
- int maxCountOfAmbariDb, i = 0;
+ int maxCountOfAmbariDb = 0, i = 0, sequence;
String time = null;
Long epochTime = null;
String dirNameforHiveHistroy;
@@ -145,6 +145,12 @@ public class HiveHistoryMigrationUtility {
// for each queries fetched from Hue database//
+ //
+ int id = 0;
+
+ id = hiveHistoryQueryImpl.fetchInstanceTablename(connectionAmbaridb, instance, ambaridatabase); // feching table name according to the given instance name
+ sequence = hiveHistoryQueryImpl.fetchSequenceno(connectionAmbaridb, id, ambaridatabase);
+ //
for (i = 0; i < dbpojoHiveHistoryQuery.size(); i++) {
float calc = ((float) (i + 1)) / dbpojoHiveHistoryQuery.size() * 100;
@@ -159,9 +165,6 @@ public class HiveHistoryMigrationUtility {
logger.info("Loop No." + (i + 1));
logger.info("_____________________");
logger.info("Hue query that has been fetched" + dbpojoHiveHistoryQuery.get(i).getQuery());
- int id = 0;
-
- id = hiveHistoryQueryImpl.fetchInstanceTablename(connectionAmbaridb, instance, ambaridatabase); // feching table name according to the given instance name
logger.info("Table name has been fetched from intance name");
@@ -173,35 +176,38 @@ public class HiveHistoryMigrationUtility {
logger.info("Log file created in Temp directory");
- maxCountOfAmbariDb = (hiveHistoryQueryImpl.fetchMaximumIdfromAmbaridb(connectionAmbaridb, id, ambaridatabase) + 1);// fetching the maximum count for ambari db to insert
+ maxCountOfAmbariDb = i + sequence + 1;
time = hiveHistoryQueryImpl.getTime();// getting the system current time.
epochTime = hiveHistoryQueryImpl.getEpochTime();// getting system time as epoch format
- dirNameforHiveHistroy = "/user/admin/migration/jobs/migration-job-" + maxCountOfAmbariDb + "-" + time + "/";// creating the directory name
+ dirNameforHiveHistroy = "/user/"+username+"/hive/jobs/hive-job-" + maxCountOfAmbariDb + "-" + time + "/";// creating the directory name
logger.info("Directory name where .hql will be saved: " + dirNameforHiveHistroy);
- hiveHistoryQueryImpl.insertRowinAmbaridb(dirNameforHiveHistroy, maxCountOfAmbariDb, epochTime, connectionAmbaridb, id, instance, i, ambaridatabase);// inserting in ambari database
+ String versionName = hiveHistoryQueryImpl.getAllHiveVersionInstance(connectionAmbaridb, ambaridatabase, instance);
+
+ hiveHistoryQueryImpl.insertRowinAmbaridb(dirNameforHiveHistroy, maxCountOfAmbariDb, epochTime, connectionAmbaridb, id, instance, i, ambaridatabase, versionName, username);// inserting in ambari database
if (view.getProperties().get("KerberoseEnabled").equals("y")) {
logger.info("kerberose enabled");
- hiveHistoryQueryImpl.createDirKerberorisedSecured(dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"));// creating directory in kerborized secured hdfs
+ hiveHistoryQueryImpl.createDirKerberorisedSecured(dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));// creating directory in kerborized secured hdfs
logger.info("Directory created in hdfs");
- hiveHistoryQueryImpl.putFileinHdfsKerborizedSecured(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"));// copying the .hql file to kerborized hdfs
- hiveHistoryQueryImpl.putFileinHdfsKerborizedSecured(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"));// copying the log file to kerborized hdfs
+ hiveHistoryQueryImpl.putFileinHdfsKerborizedSecured(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));// copying the .hql file to kerborized hdfs
+ hiveHistoryQueryImpl.putFileinHdfsKerborizedSecured(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));// copying the log file to kerborized hdfs
} else {
logger.info("kerberose not enabled");
- hiveHistoryQueryImpl.createDir(dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"));// creating directory in hdfs
+ hiveHistoryQueryImpl.createDir(dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"),username);// creating directory in hdfs
logger.info("Directory created in hdfs");
- hiveHistoryQueryImpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"));// copying the .hql file to hdfs
- hiveHistoryQueryImpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"));// copying the log file to hdfs
+ hiveHistoryQueryImpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"),username);// copying the .hql file to hdfs
+ hiveHistoryQueryImpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveHistroy, view.getProperties().get("namenode_URI_Ambari"),username);// copying the log file to hdfs
}
}
+ hiveHistoryQueryImpl.updateSequenceno(connectionAmbaridb, maxCountOfAmbariDb, id, ambaridatabase);
connectionAmbaridb.commit();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryQueryMigrationImplementation.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryQueryMigrationImplementation.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryQueryMigrationImplementation.java
index a0182f6..55c81c0 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryQueryMigrationImplementation.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/historyquery/HiveHistoryQueryMigrationImplementation.java
@@ -36,16 +36,15 @@ import org.jdom.input.SAXBuilder;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
+import java.beans.PropertyVetoException;
import java.io.*;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.sql.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
+import java.util.*;
import java.util.Date;
-import java.util.GregorianCalendar;
public class HiveHistoryQueryMigrationImplementation {
@@ -131,28 +130,79 @@ public class HiveHistoryQueryMigrationImplementation {
}
- public void insertRowinAmbaridb(String dirname, int maxcount, long epochtime, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+ public void insertRowinAmbaridb(String dirname, int maxcount, long epochtime, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase, String versionName, String username) throws SQLException, IOException {
String maxcount1 = Integer.toString(maxcount);
String epochtime1 = Long.toString(epochtime);
PreparedStatement prSt = null;
String revsql = null;
+ if (versionName.contains("1.5")) {
+ prSt = ambaridatabase.insertToHiveHistoryForHiveNext(c, id, maxcount1, epochtime, dirname, username);
+ }
+ if (versionName.contains("1.0")) {
+ prSt = ambaridatabase.insertToHiveHistoryForHive(c, id, maxcount1, epochtime, dirname, username);
+ }
+ logger.info("The actual insert statement is " + prSt);
+ prSt.executeUpdate();
+ revsql = ambaridatabase.revertSql(id, maxcount1);
+ logger.info("adding revert sql hive history");
+ wrtitetoalternatesqlfile(dirname, revsql, instance, i);
+ }
- prSt = ambaridatabase.insertToHiveHistory(c, id, maxcount1, epochtime, dirname);
+ public void updateSequenceno(Connection c, int seqNo, int id, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+ PreparedStatement prSt;
+ prSt = ambaridatabase.updateSequenceNoInAmbariSequence(c, seqNo, id);
logger.info("The actual insert statement is " + prSt);
-
prSt.executeUpdate();
+ logger.info("adding revert sql hive history");
+ }
- revsql = ambaridatabase.RevertSql(id, maxcount1);
+ public String getAllHiveVersionInstance(Connection c, QuerySetAmbariDB ambaridatabase, String viewName) throws PropertyVetoException, SQLException, IOException {
- logger.info("adding revert sql hive history");
- wrtitetoalternatesqlfile(dirname, revsql, instance, i);
+ PreparedStatement prSt;
+ ResultSet rs1 = null;
+ String instanceVersion = null;
+ int i = 0;
+ prSt = ambaridatabase.getHiveVersionInstance(c, viewName);
+ rs1 = prSt.executeQuery();
+
+ while (rs1.next()) {
+ instanceVersion = rs1.getString(1);
+
+ }
+ rs1.close();
+ prSt.close();
+ return instanceVersion;
}
+ public int fetchSequenceno(Connection c, int id, QuerySetAmbariDB ambaridatabase) throws SQLException {
+
+ String ds_id = new String();
+ Statement stmt = null;
+ PreparedStatement prSt = null;
+ int sequencevalue = 0;
+
+
+ ResultSet rs = null;
+
+
+ prSt = ambaridatabase.getSequenceNoFromAmbariSequence(c, id);
+
+ logger.info("sql statement to fetch is from ambari instance:= = " + prSt);
+
+ rs = prSt.executeQuery();
+
+ while (rs.next()) {
+ sequencevalue = rs.getInt("sequence_value");
+ }
+ return sequencevalue;
+ }
+
+
public int fetchInstanceTablename(Connection c, String instance, QuerySetAmbariDB ambaridatabase) throws SQLException {
String ds_id = new String();
@@ -177,7 +227,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
public long getEpochTime() throws ParseException {
- long seconds = System.currentTimeMillis() / 1000l;
+ long seconds = System.currentTimeMillis();
return seconds;
}
@@ -364,7 +414,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
- public void createDir(final String dir, final String namenodeuri) throws IOException,
+ public void createDir(final String dir, final String namenodeuri, final String username) throws IOException,
URISyntaxException {
try {
@@ -387,8 +437,10 @@ public class HiveHistoryQueryMigrationImplementation {
public Boolean run() throws Exception {
FileSystem fs = FileSystem.get(conf);
+
Path src = new Path(dir);
Boolean b = fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
return b;
}
});
@@ -397,7 +449,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
}
- public void createDirKerberorisedSecured(final String dir, final String namenodeuri) throws IOException,
+ public void createDirKerberorisedSecured(final String dir, final String namenodeuri, final String username,final String principalName) throws IOException,
URISyntaxException {
try {
@@ -410,16 +462,18 @@ public class HiveHistoryQueryMigrationImplementation {
org.apache.hadoop.fs.LocalFileSystem.class.getName()
);
conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
conf.set("hadoop.security.authentication", "Kerberos");
UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
public Boolean run() throws Exception {
FileSystem fs = FileSystem.get(conf);
Path src = new Path(dir);
Boolean b = fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
return b;
}
});
@@ -429,7 +483,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
- public void putFileinHdfs(final String source, final String dest, final String namenodeuri)
+ public void putFileinHdfs(final String source, final String dest, final String namenodeuri,final String username)
throws IOException {
try {
@@ -476,6 +530,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
in.close();
out.close();
+ fileSystem.setOwner(path,username,"hadoop");
fileSystem.close();
return null;
}
@@ -486,7 +541,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
- public void putFileinHdfsKerborizedSecured(final String source, final String dest, final String namenodeuri)
+ public void putFileinHdfsKerborizedSecured(final String source, final String dest, final String namenodeuri,final String username,final String principalName)
throws IOException {
try {
@@ -500,11 +555,11 @@ public class HiveHistoryQueryMigrationImplementation {
org.apache.hadoop.fs.LocalFileSystem.class.getName()
);
conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
conf.set("hadoop.security.authentication", "Kerberos");
UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
-
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
ugi.doAs(new PrivilegedExceptionAction<Void>() {
public Void run() throws Exception {
@@ -537,6 +592,7 @@ public class HiveHistoryQueryMigrationImplementation {
}
in.close();
out.close();
+ fileSystem.setOwner(path,username,"hadoop");
fileSystem.close();
return null;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationImplementation.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationImplementation.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationImplementation.java
index 7bd48b2..4d9a6c3 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationImplementation.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationImplementation.java
@@ -150,6 +150,39 @@ public class HiveSavedQueryMigrationImplementation {
return num;
}
+
+ public int fetchSequenceno(Connection c, int id, QuerySetAmbariDB ambaridatabase) throws SQLException {
+
+ String ds_id = new String();
+ Statement stmt = null;
+ PreparedStatement prSt = null;
+ int sequencevalue=0;
+
+
+ ResultSet rs = null;
+
+
+ prSt = ambaridatabase.getSequenceNoFromAmbariSequence(c, id);
+
+ logger.info("sql statement to fetch is from ambari instance:= = " + prSt);
+
+ rs = prSt.executeQuery();
+
+ while (rs.next()) {
+ sequencevalue = rs.getInt("sequence_value");
+ }
+ return sequencevalue;
+ }
+
+ public void updateSequenceno(Connection c, int seqNo, int id, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+
+ PreparedStatement prSt;
+ prSt = ambaridatabase.updateSequenceNoInAmbariSequence(c, seqNo, id);
+ logger.info("The actual insert statement is " + prSt);
+ prSt.executeUpdate();
+ logger.info("adding revert sql hive history");
+ }
+
public int fetchInstancetablenameForSavedqueryHive(Connection c, String instance, QuerySetAmbariDB ambaridatabase) throws SQLException {
String ds_id = new String();
@@ -245,14 +278,14 @@ public class HiveSavedQueryMigrationImplementation {
wrtitetoalternatesqlfile(dirname, revsql, instance, i);
}
- public void insertRowinSavedQuery(int maxcount, String database, String dirname, String query, String name, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+ public void insertRowinSavedQuery(int maxcount, String database, String dirname, String query, String name, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase,String username) throws SQLException, IOException {
String maxcount1 = Integer.toString(maxcount);
String revsql = null;
PreparedStatement prSt = null;
- prSt = ambaridatabase.insertToHiveSavedQuery(c, id, maxcount1, database, dirname, query, name);
+ prSt = ambaridatabase.insertToHiveSavedQuery(c, id, maxcount1, database, dirname, query, name,username);
System.out.println("the actual query is " + prSt);
@@ -480,7 +513,7 @@ public class HiveSavedQueryMigrationImplementation {
}
- public void createDirHive(final String dir, final String namenodeuri)
+ public void createDirHive(final String dir, final String namenodeuri,final String username)
throws IOException, URISyntaxException {
try {
@@ -506,6 +539,7 @@ public class HiveSavedQueryMigrationImplementation {
FileSystem fs = FileSystem.get(conf);
Path src = new Path(dir);
fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
return null;
}
});
@@ -514,9 +548,8 @@ public class HiveSavedQueryMigrationImplementation {
}
}
- public void createDirHiveSecured(final String dir, final String namenodeuri)
+ public void createDirHiveSecured(final String dir, final String namenodeuri,final String username,final String principalName)
throws IOException, URISyntaxException {
-
try {
final Configuration conf = new Configuration();
@@ -527,29 +560,28 @@ public class HiveSavedQueryMigrationImplementation {
org.apache.hadoop.fs.LocalFileSystem.class.getName()
);
conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
conf.set("hadoop.security.authentication", "Kerberos");
-
UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
+ ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+ public Boolean run() throws Exception {
FileSystem fs = FileSystem.get(conf);
Path src = new Path(dir);
- fs.mkdirs(src);
- return null;
+ Boolean b = fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
+ return b;
}
});
} catch (Exception e) {
- logger.error("Webhdfs: ", e);
+ logger.error("Exception in Webhdfs", e);
}
}
public void putFileinHdfs(final String source, final String dest,
- final String namenodeuri) throws IOException {
+ final String namenodeuri,final String username) throws IOException {
try {
final Configuration conf = new Configuration();
@@ -596,6 +628,7 @@ public class HiveSavedQueryMigrationImplementation {
}
in.close();
out.close();
+ fileSystem.setOwner(path,username,"hadoop");
fileSystem.close();
return null;
}
@@ -607,10 +640,10 @@ public class HiveSavedQueryMigrationImplementation {
}
- public void putFileinHdfsSecured(final String source, final String dest,
- final String namenodeuri) throws IOException {
+ public void putFileinHdfsSecured(final String source, final String dest,final String namenodeuri,final String username,final String principalName) throws IOException {
try {
+
final Configuration conf = new Configuration();
conf.set("fs.hdfs.impl",
@@ -620,16 +653,15 @@ public class HiveSavedQueryMigrationImplementation {
org.apache.hadoop.fs.LocalFileSystem.class.getName()
);
conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
conf.set("hadoop.security.authentication", "Kerberos");
-
UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
ugi.doAs(new PrivilegedExceptionAction<Void>() {
public Void run() throws Exception {
-
FileSystem fileSystem = FileSystem.get(conf);
String filename = source.substring(
@@ -645,7 +677,7 @@ public class HiveSavedQueryMigrationImplementation {
if (fileSystem.exists(path)) {
}
- // Path pathsource = new Path(source);
+
FSDataOutputStream out = fileSystem.create(path);
InputStream in = new BufferedInputStream(
@@ -658,14 +690,14 @@ public class HiveSavedQueryMigrationImplementation {
}
in.close();
out.close();
+ fileSystem.setOwner(path,username,"hadoop");
fileSystem.close();
-
-
return null;
}
});
} catch (Exception e) {
logger.error("Webhdfs exception", e);
+
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
index 9f9e053..566b369 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
@@ -47,7 +47,6 @@ import java.util.ArrayList;
public class HiveSavedQueryMigrationUtility {
-
protected MigrationResourceManager resourceManager = null;
public synchronized PersonalCRUDResourceManager<MigrationResponse> getResourceManager(ViewContext view) {
@@ -78,43 +77,31 @@ public class HiveSavedQueryMigrationUtility {
HiveSavedQueryMigrationImplementation hivesavedqueryimpl = new HiveSavedQueryMigrationImplementation();/* creating Implementation object */
- QuerySet huedatabase=null;
+ QuerySet huedatabase = null;
- if(view.getProperties().get("huedrivername").contains("mysql"))
- {
- huedatabase=new MysqlQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("postgresql"))
- {
- huedatabase=new PostgressQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("sqlite"))
- {
- huedatabase=new SqliteQuerySet();
- }
- else if (view.getProperties().get("huedrivername").contains("oracle"))
- {
- huedatabase=new OracleQuerySet();
+ if (view.getProperties().get("huedrivername").contains("mysql")) {
+ huedatabase = new MysqlQuerySet();
+ } else if (view.getProperties().get("huedrivername").contains("postgresql")) {
+ huedatabase = new PostgressQuerySet();
+ } else if (view.getProperties().get("huedrivername").contains("sqlite")) {
+ huedatabase = new SqliteQuerySet();
+ } else if (view.getProperties().get("huedrivername").contains("oracle")) {
+ huedatabase = new OracleQuerySet();
}
- QuerySetAmbariDB ambaridatabase=null;
+ QuerySetAmbariDB ambaridatabase = null;
- if(view.getProperties().get("ambaridrivername").contains("mysql"))
- {
- ambaridatabase=new MysqlQuerySetAmbariDB();
- }
- else if(view.getProperties().get("ambaridrivername").contains("postgresql"))
- {
- ambaridatabase=new PostgressQuerySetAmbariDB();
- }
- else if (view.getProperties().get("ambaridrivername").contains("oracle"))
- {
- ambaridatabase= new OracleQuerySetAmbariDB();
+ if (view.getProperties().get("ambaridrivername").contains("mysql")) {
+ ambaridatabase = new MysqlQuerySetAmbariDB();
+ } else if (view.getProperties().get("ambaridrivername").contains("postgresql")) {
+ ambaridatabase = new PostgressQuerySetAmbariDB();
+ } else if (view.getProperties().get("ambaridrivername").contains("oracle")) {
+ ambaridatabase = new OracleQuerySetAmbariDB();
}
- int maxcountForHivehistroryAmbaridb, maxCountforSavequeryAmbaridb;
+ int maxcountForHivehistroryAmbaridb, maxCountforSavequeryAmbaridb = 0;
String time = null;
Long epochtime = null;
String dirNameforHiveSavedquery;
@@ -124,12 +111,11 @@ public class HiveSavedQueryMigrationUtility {
connectionHuedb = DataSourceHueDatabase.getInstance(view.getProperties().get("huedrivername"), view.getProperties().get("huejdbcurl"), view.getProperties().get("huedbusername"), view.getProperties().get("huedbpassword")).getConnection(); /* fetching connection to hue DB */
- dbpojoHiveSavedQuery = hivesavedqueryimpl.fetchFromHuedb(username, startDate, endDate, connectionHuedb,huedatabase); /* fetching data from hue db and storing it in to a model */
+ dbpojoHiveSavedQuery = hivesavedqueryimpl.fetchFromHuedb(username, startDate, endDate, connectionHuedb, huedatabase); /* fetching data from hue db and storing it in to a model */
- for(int j=0;j<dbpojoHiveSavedQuery.size();j++)
- {
- logger.info("the query fetched from hue"+dbpojoHiveSavedQuery.get(j).getQuery());
+ for (int j = 0; j < dbpojoHiveSavedQuery.size(); j++) {
+ logger.info("the query fetched from hue" + dbpojoHiveSavedQuery.get(j).getQuery());
}
@@ -151,6 +137,9 @@ public class HiveSavedQueryMigrationUtility {
connectionAmbaridb = DataSourceAmbariDatabase.getInstance(view.getProperties().get("ambaridrivername"), view.getProperties().get("ambarijdbcurl"), view.getProperties().get("ambaridbusername"), view.getProperties().get("ambaridbpassword")).getConnection();/* connecting to ambari DB */
connectionAmbaridb.setAutoCommit(false);
+ int tableIdSavedQuery = hivesavedqueryimpl.fetchInstancetablenameForSavedqueryHive(connectionAmbaridb, instance, ambaridatabase); /* fetching the instance table name for migration saved query from the given instance name */
+ int sequence = hivesavedqueryimpl.fetchSequenceno(connectionAmbaridb, tableIdSavedQuery, ambaridatabase);
+
for (i = 0; i < dbpojoHiveSavedQuery.size(); i++) {
logger.info("_____________________");
@@ -162,19 +151,12 @@ public class HiveSavedQueryMigrationUtility {
migrationresult.setIsNoQuerySelected("no");
migrationresult.setProgressPercentage(progressPercentage);
- migrationresult.setNumberOfQueryTransfered(i+1);
+ migrationresult.setNumberOfQueryTransfered(i + 1);
migrationresult.setTotalNoQuery(dbpojoHiveSavedQuery.size());
getResourceManager(view).update(migrationresult, jobid);
-
-
-
logger.info("query fetched from hue:- " + dbpojoHiveSavedQuery.get(i).getQuery());
- int tableIdSavedQuery = hivesavedqueryimpl.fetchInstancetablenameForSavedqueryHive(connectionAmbaridb, instance,ambaridatabase); /* fetching the instance table name for migration saved query from the given instance name */
-
- int tableIdHistoryHive = hivesavedqueryimpl.fetchInstanceTablenameHiveHistory(connectionAmbaridb, instance,ambaridatabase); /* fetching the instance table name for migration history query from the given instance name */
-
logger.info("Table name are fetched from instance name.");
hivesavedqueryimpl.writetoFilequeryHql(dbpojoHiveSavedQuery.get(i).getQuery(), ConfigurationCheckImplementation.getHomeDir()); /* writing migration query to a local file*/
@@ -183,43 +165,37 @@ public class HiveSavedQueryMigrationUtility {
logger.info(".hql and logs file are saved in temporary directory");
- maxcountForHivehistroryAmbaridb = (hivesavedqueryimpl.fetchMaxdsidFromHiveHistory( connectionAmbaridb, tableIdHistoryHive,ambaridatabase) + 1);/* fetching the maximum ds_id from migration history table*/
-
- maxCountforSavequeryAmbaridb = (hivesavedqueryimpl.fetchMaxidforSavedQueryHive(connectionAmbaridb, tableIdSavedQuery,ambaridatabase) + 1);/* fetching the maximum ds_id from migration saved query table*/
+ maxCountforSavequeryAmbaridb = i + sequence + 1;
time = hivesavedqueryimpl.getTime();/* getting system time */
- epochtime = hivesavedqueryimpl.getEpochTime();/* getting epoch time */
-
- dirNameforHiveSavedquery = "/user/admin/migration/jobs/migration-job-" + maxcountForHivehistroryAmbaridb + "-"
+ dirNameforHiveSavedquery = "/user/"+username+"/hive/scripts/hive-query-" + maxCountforSavequeryAmbaridb + "-"
+ time + "/"; // creating hdfs directory name
logger.info("Directory will be creted in HDFS" + dirNameforHiveSavedquery);
- hivesavedqueryimpl.insertRowHiveHistory(dirNameforHiveSavedquery,maxcountForHivehistroryAmbaridb,epochtime,connectionAmbaridb,tableIdHistoryHive,instance,i,ambaridatabase);// inserting to migration history table
-
logger.info("Row inserted in hive History table.");
if (view.getProperties().get("KerberoseEnabled").equals("y")) {
logger.info("Kerberose Enabled");
- hivesavedqueryimpl.createDirHiveSecured(dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// creating directory in hdfs in kerborized cluster
- hivesavedqueryimpl.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting .hql file in hdfs in kerberoroized cluster
- hivesavedqueryimpl.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting logs file in hdfs in kerberoroized cluster
+ hivesavedqueryimpl.createDirHiveSecured(dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));// creating directory in hdfs in kerborized cluster
+ hivesavedqueryimpl.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));// putting .hql file in hdfs in kerberoroized cluster
+ hivesavedqueryimpl.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"),username,view.getProperties().get("PrincipalUserName"));// putting logs file in hdfs in kerberoroized cluster
} else {
-
logger.info("Kerberose Not Enabled");
- hivesavedqueryimpl.createDirHive(dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// creating directory in hdfs
- hivesavedqueryimpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting .hql file in hdfs directory
- hivesavedqueryimpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting logs file in hdfs
+ hivesavedqueryimpl.createDirHive(dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"),username);// creating directory in hdfs
+ hivesavedqueryimpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"),username);// putting .hql file in hdfs directory
+ hivesavedqueryimpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"),username);// putting logs file in hdfs
}
//inserting into hived saved query table
//6.
- hivesavedqueryimpl.insertRowinSavedQuery(maxCountforSavequeryAmbaridb, dbpojoHiveSavedQuery.get(i).getDatabase(), dirNameforHiveSavedquery, dbpojoHiveSavedQuery.get(i).getQuery(), dbpojoHiveSavedQuery.get(i).getOwner(), connectionAmbaridb, tableIdSavedQuery, instance, i,ambaridatabase);
+ hivesavedqueryimpl.insertRowinSavedQuery(maxCountforSavequeryAmbaridb, dbpojoHiveSavedQuery.get(i).getDatabase(), dirNameforHiveSavedquery, dbpojoHiveSavedQuery.get(i).getQuery(), dbpojoHiveSavedQuery.get(i).getOwner(), connectionAmbaridb, tableIdSavedQuery, instance, i, ambaridatabase, username);
}
+ hivesavedqueryimpl.updateSequenceno(connectionAmbaridb, maxCountforSavequeryAmbaridb, tableIdSavedQuery, ambaridatabase);
connectionAmbaridb.commit();
}
@@ -236,13 +212,13 @@ public class HiveSavedQueryMigrationUtility {
}
} catch (ClassNotFoundException e1) {
- logger.error("Class not found : " , e1);
+ logger.error("Class not found : ", e1);
} catch (ParseException e) {
- logger.error("ParseException: " , e);
+ logger.error("ParseException: ", e);
} catch (URISyntaxException e) {
- logger.error("URISyntaxException: " , e);
+ logger.error("URISyntaxException: ", e);
} catch (PropertyVetoException e) {
- logger.error("PropertyVetoException:" , e);
+ logger.error("PropertyVetoException:", e);
} finally {
if (null != connectionAmbaridb)
try {
@@ -259,14 +235,13 @@ public class HiveSavedQueryMigrationUtility {
long stopTime = System.currentTimeMillis();
long elapsedTime = stopTime - startTime;
- MigrationModel model=new MigrationModel();
+ MigrationModel model = new MigrationModel();
migrationresult.setJobtype("hivesavedquerymigration");
migrationresult.setTotalTimeTaken(String.valueOf(elapsedTime));
getResourceManager(view).update(migrationresult, jobid);
-
logger.info("-------------------------------");
logger.info("hive saved query Migration end");
logger.info("--------------------------------");
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
index 0445132..de81f0b 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
@@ -44,8 +44,6 @@ public class HiveSavedQueryStartJob extends Thread{
this.view=view;
}
-
-
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a34fe1d/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
index 64e7069..8f843d0 100644
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
+++ b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
@@ -32,8 +32,8 @@ import java.util.GregorianCalendar;
import java.io.*;
import java.net.URISyntaxException;
;
+import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.jobqueryset.QuerySetAmbariDB;
import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.pig.jobqueryset.QuerySet;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.jobqueryset.*;
import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.PigModel;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -141,6 +141,29 @@ public class PigJobMigrationImplementation {
}
+ public int fetchSequenceno(Connection c, int id, QuerySetAmbariDB ambaridatabase) throws SQLException {
+
+ String ds_id = new String();
+ Statement stmt = null;
+ PreparedStatement prSt = null;
+ int sequencevalue=0;
+
+
+ ResultSet rs = null;
+
+
+ prSt = ambaridatabase.getSequenceNoFromAmbariSequence(c, id);
+
+ logger.info("sql statement to fetch is from ambari instance:= = " + prSt);
+
+ rs = prSt.executeQuery();
+
+ while (rs.next()) {
+ sequencevalue = rs.getInt("sequence_value");
+ }
+ return sequencevalue;
+ }
+
public int fetchInstanceTablename(Connection c, String instance, QuerySetAmbariDB ambaridatabase) throws SQLException {
@@ -165,7 +188,7 @@ public class PigJobMigrationImplementation {
return id;
}
- public void insertRowPigJob(String dirname, int maxcountforpigjob, String time, String time2, long epochtime, String title, Connection c, int id, String status, String instance, int i, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+ public void insertRowPigJob(String dirname, int maxcountforpigjob, String time, String time2, long epochtime, String title, Connection c, int id, String status, String instance, int i, QuerySetAmbariDB ambaridatabase,String username) throws SQLException, IOException {
String epochtime1 = Long.toString(epochtime);
String maxcountforpigjob1 = Integer.toString(maxcountforpigjob);
@@ -174,7 +197,7 @@ public class PigJobMigrationImplementation {
PreparedStatement prSt = null;
- prSt = ambaridatabase.insertToPigJob(dirname, maxcountforpigjob1, epochtime, title, c, id, status);
+ prSt = ambaridatabase.insertToPigJob(dirname, maxcountforpigjob1, epochtime, title, c, id, status,username);
prSt.executeUpdate();
@@ -341,7 +364,23 @@ public class PigJobMigrationImplementation {
}
- public void createDirPigJob(final String dir, final String namenodeuri) throws IOException,
+ public void updateSequenceno(Connection c, int seqNo, int id, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
+
+ PreparedStatement prSt;
+
+ prSt = ambaridatabase.updateSequenceNoInAmbariSequence(c, seqNo, id);
+
+ logger.info("The actual insert statement is " + prSt);
+
+ prSt.executeUpdate();
+
+
+ logger.info("adding revert sql hive history");
+
+
+
+ }
+ public void createDirPigJob(final String dir, final String namenodeuri,final String username) throws IOException,
URISyntaxException {
try {
@@ -365,6 +404,7 @@ public class PigJobMigrationImplementation {
FileSystem fs = FileSystem.get(conf);
Path src = new Path(dir);
fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
return null;
}
});
@@ -374,9 +414,8 @@ public class PigJobMigrationImplementation {
}
/**/
- public void createDirPigJobSecured(final String dir, final String namenodeuri) throws IOException,
+ public void createDirPigJobSecured(final String dir, final String namenodeuri,final String username,final String principalName) throws IOException,
URISyntaxException {
-
try {
final Configuration conf = new Configuration();
@@ -387,30 +426,28 @@ public class PigJobMigrationImplementation {
org.apache.hadoop.fs.LocalFileSystem.class.getName()
);
conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
conf.set("hadoop.security.authentication", "Kerberos");
-
UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(principalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
+ ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+ public Boolean run() throws Exception {
FileSystem fs = FileSystem.get(conf);
Path src = new Path(dir);
- fs.mkdirs(src);
- return null;
+ Boolean b = fs.mkdirs(src);
+ fs.setOwner(src,username,"hadoop");
+ return b;
}
});
} catch (Exception e) {
- logger.error("Webhdfs exception: ", e);
+ logger.error("Exception in Webhdfs", e);
}
}
/**/
- public void copyFileBetweenHdfs(final String source, final String dest, final String nameNodeuriAmbari, final String nameNodeuriHue)
+ public void copyFileBetweenHdfs(final String source, final String dest, final String nameNodeuriAmbari, final String nameNodeuriHue,final String username)
throws IOException {
try {
@@ -457,6 +494,7 @@ public class PigJobMigrationImplementation {
}
in1.close();
out.close();
+ fileSystemAmbari.setOwner(path,username,"hadoop");
fileSystemAmbari.close();
return null;
}
@@ -468,7 +506,7 @@ public class PigJobMigrationImplementation {
}
/**/
- public void copyFileBetweenHdfsSecured(final String source, final String dest, final String nameNodeuriAmbari, final String nameNodeuriHue)
+ public void copyFileBetweenHdfsSecured(final String source, final String dest, final String nameNodeuriAmbari, final String nameNodeuriHue,final String username,final String pricipalName)
throws IOException {
try {
@@ -484,8 +522,10 @@ public class PigJobMigrationImplementation {
confAmbari.set("hadoop.security.authentication", "Kerberos");
confHue.set("hadoop.security.authentication", "Kerberos");
- UserGroupInformation ugi = UserGroupInformation
- .createRemoteUser("hdfs");
+ UserGroupInformation.setConfiguration(confAmbari);
+ UserGroupInformation proxyUser ;
+ proxyUser = UserGroupInformation.createRemoteUser(pricipalName);
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser("hdfs", proxyUser);
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@@ -519,6 +559,7 @@ public class PigJobMigrationImplementation {
}
in1.close();
out.close();
+ fileSystemAmbari.setOwner(path,username,"hadoop");
fileSystemAmbari.close();
return null;
}
[49/55] [abbrv] ambari git commit: AMBARI-18212. Log entry missing in
the preview log modal window (Dharmesh Makwana via oleewere)
Posted by ol...@apache.org.
AMBARI-18212. Log entry missing in the preview log modal window (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a971c1e8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a971c1e8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a971c1e8
Branch: refs/heads/branch-dev-logsearch
Commit: a971c1e8ed1ac0ee2a7dba619f2c081d964119cc
Parents: 7ebdca6
Author: oleewere <ol...@gmail.com>
Authored: Mon Aug 22 11:00:31 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:03 2016 +0200
----------------------------------------------------------------------
.../src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a971c1e8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
index 451fac1..a2c15f4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java
@@ -1886,7 +1886,7 @@ public class LogsMgr extends MgrBase {
String order1 = LogSearchConstants.LOGTIME + " "
+ LogSearchConstants.DESCENDING_ORDER;
String order2 = LogSearchConstants.SEQUNCE_ID + " "
- + LogSearchConstants.ASCENDING_ORDER;
+ + LogSearchConstants.DESCENDING_ORDER;
List<String> sortOrder = new ArrayList<String>();
sortOrder.add(order1);
sortOrder.add(order2);
[10/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
Revert "AMBARI-18156 Remove upgradeHelper.py and corresponding files due to drop of the manual upgrade (dsen)"
This reverts commit 793adfe46a02e0870ae8aa4f334818d5fa401965.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c758b7ff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c758b7ff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c758b7ff
Branch: refs/heads/branch-dev-logsearch
Commit: c758b7ffe98f136590012867d4a2bba1484c051e
Parents: 8139383
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Thu Aug 18 16:14:55 2016 +0200
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Thu Aug 18 16:14:55 2016 +0200
----------------------------------------------------------------------
ambari-server/src/main/python/upgradeHelper.py | 2338 ++++++++++++++++++
.../catalog/UpgradeCatalog_1.3_to_2.2.json | 948 +++++++
.../catalog/UpgradeCatalog_2.0_to_2.2.2.json | 408 +++
.../catalog/UpgradeCatalog_2.0_to_2.2.4.json | 453 ++++
.../catalog/UpgradeCatalog_2.0_to_2.2.json | 275 ++
.../catalog/UpgradeCatalog_2.1_to_2.2.2.json | 465 ++++
.../catalog/UpgradeCatalog_2.1_to_2.2.4.json | 499 ++++
.../catalog/UpgradeCatalog_2.1_to_2.2.json | 292 +++
.../catalog/UpgradeCatalog_2.1_to_2.3.json | 440 ++++
.../catalog/UpgradeCatalog_2.2_to_2.3.json | 2234 +++++++++++++++++
.../UpgradeCatalog_2.2_to_2.3_step2.json | 81 +
.../src/test/python/TestUpgradeHelper.py | 1028 ++++++++
12 files changed, 9461 insertions(+)
----------------------------------------------------------------------
[31/55] [abbrv] ambari git commit: AMBARI-18207. For atlas,
solr configSet should be renamed from "basic_configs" to
"atlas_configs" (Ayub Khan via alejandro)
Posted by ol...@apache.org.
AMBARI-18207. For atlas, solr configSet should be renamed from "basic_configs" to "atlas_configs" (Ayub Khan via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/48831ba5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/48831ba5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/48831ba5
Branch: refs/heads/branch-dev-logsearch
Commit: 48831ba56e8adaa1840eb9efa0651265883c712d
Parents: da07ea1
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Fri Aug 19 15:24:52 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Fri Aug 19 15:24:53 2016 -0700
----------------------------------------------------------------------
.../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 8 ++--
.../stacks/2.3/ATLAS/test_metadata_server.py | 44 ++++++++++----------
.../stacks/2.5/ATLAS/test_atlas_server.py | 22 +++++-----
3 files changed, 37 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/48831ba5/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 2ae0c6e..a195fd3 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -116,11 +116,11 @@ def metadata(type='server'):
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile=params.atlas_jaas_file if params.security_enabled else None
- upload_conf_set('basic_configs', jaasFile)
+ upload_conf_set('atlas_configs', jaasFile)
- create_collection('vertex_index', 'basic_configs', jaasFile)
- create_collection('edge_index', 'basic_configs', jaasFile)
- create_collection('fulltext_index', 'basic_configs', jaasFile)
+ create_collection('vertex_index', 'atlas_configs', jaasFile)
+ create_collection('edge_index', 'atlas_configs', jaasFile)
+ create_collection('fulltext_index', 'atlas_configs', jaasFile)
File(params.atlas_hbase_setup,
content=Template("atlas_hbase_setup.rb.j2")
http://git-wip-us.apache.org/repos/asf/ambari/blob/48831ba5/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index 0f7ed2f..f6886ee 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -151,21 +151,21 @@ class TestMetadataServer(RMFTestCase):
content=''
)
self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_atlas_configs_0.[0-9]* --config-set atlas_configs --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_atlas_configs_0.[0-9]*',
content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']),
- only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5',
- only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5',
- not_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ only_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_atlas_configs_0.[0-9]* --config-set atlas_configs --retry 30 --interval 5',
+ only_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set atlas_configs --retry 30 --interval 5',
+ not_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_atlas_configs_0.[0-9]*',
action=['delete'],
create_parents=True)
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
def configureResourcesCalledSecure(self):
# Both server and client
@@ -294,21 +294,21 @@ class TestMetadataServer(RMFTestCase):
content=''
)
self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_atlas_configs_0.[0-9]* --config-set atlas_configs --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_atlas_configs_0.[0-9]*',
content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']),
- only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5',
- only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5',
- not_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ only_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_atlas_configs_0.[0-9]* --config-set atlas_configs --retry 30 --interval 5',
+ only_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set atlas_configs --retry 30 --interval 5',
+ not_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_atlas_configs_0.[0-9]*',
action=['delete'],
create_parents=True)
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
http://git-wip-us.apache.org/repos/asf/ambari/blob/48831ba5/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index b256bb2..8e84d2a 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -150,20 +150,20 @@ class TestAtlasServer(RMFTestCase):
)
self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181 --znode /infra-solr --check-znode --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5')
- self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --download-config --config-dir /tmp/solr_config_atlas_configs_0.[0-9]* --config-set atlas_configs --retry 30 --interval 5')
+ self.assertResourceCalledRegexp('^File$', '^/tmp/solr_config_atlas_configs_0.[0-9]*',
content=InlineTemplate(self.getConfig()['configurations']['atlas-solrconfig']['content']),
- only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_basic_configs_0.[0-9]* --config-set basic_configs --retry 30 --interval 5',
- only_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set basic_configs --retry 30 --interval 5',
- not_if='test -d /tmp/solr_config_basic_configs_0.[0-9]*')
- self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_basic_configs_0.[0-9]*',
+ only_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /tmp/solr_config_atlas_configs_0.[0-9]* --config-set atlas_configs --retry 30 --interval 5',
+ only_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --upload-config --config-dir /etc/atlas/conf/solr --config-set atlas_configs --retry 30 --interval 5',
+ not_if='test -d /tmp/solr_config_atlas_configs_0.[0-9]*')
+ self.assertResourceCalledRegexp('^Directory$', '^/tmp/solr_config_atlas_configs_0.[0-9]*',
action=['delete'],
create_parents=True)
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
- self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set basic_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection vertex_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection edge_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+ self.assertResourceCalledRegexp('^Execute$', '^ambari-sudo.sh JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /usr/lib/ambari-infra-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/infra-solr --create-collection --collection fulltext_index --config-set atlas_configs --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
def test_configure_default(self):
[12/55] [abbrv] ambari git commit: AMBARI-18200. Add Service button
is not disabled if all services are installed (akovalenko)
Posted by ol...@apache.org.
AMBARI-18200. Add Service button is not disabled if all services are installed (akovalenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d3bac1d4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d3bac1d4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d3bac1d4
Branch: refs/heads/branch-dev-logsearch
Commit: d3bac1d45e9bf57d562b447d1d4540ac867ad602
Parents: 2bc8690
Author: Aleksandr Kovalenko <ak...@hortonworks.com>
Authored: Thu Aug 18 19:02:31 2016 +0300
Committer: Aleksandr Kovalenko <ak...@hortonworks.com>
Committed: Thu Aug 18 19:02:31 2016 +0300
----------------------------------------------------------------------
ambari-web/app/controllers/main/service.js | 8 +++++--
ambari-web/app/styles/application.less | 4 ----
.../main/service/all_services_actions.hbs | 2 +-
.../test/controllers/main/service_test.js | 24 ++++++++++++++++----
4 files changed, 27 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3bac1d4/ambari-web/app/controllers/main/service.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service.js b/ambari-web/app/controllers/main/service.js
index 7fe5b68..d6fbfbb 100644
--- a/ambari-web/app/controllers/main/service.js
+++ b/ambari-web/app/controllers/main/service.js
@@ -51,8 +51,12 @@ App.MainServiceController = Em.ArrayController.extend({
*/
isAllServicesInstalled: function () {
if (!this.get('content')) return false;
- var availableServices = App.StackService.find().mapProperty('serviceName');
- return this.get('content').length == availableServices.length;
+ var notAvailableServices = App.ServiceSimple.find().filterProperty('doNotShowAndInstall').mapProperty('name');
+ var availableServices = App.ServiceSimple.find().filterProperty('doNotShowAndInstall', false);
+ var installedServices = this.get('content').filter(function (service) {
+ return !notAvailableServices.contains(service.get('serviceName'));
+ });
+ return installedServices.length == availableServices.length;
}.property('content.@each', 'content.length'),
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3bac1d4/ambari-web/app/styles/application.less
----------------------------------------------------------------------
diff --git a/ambari-web/app/styles/application.less b/ambari-web/app/styles/application.less
index 2449443..3a41cd2 100644
--- a/ambari-web/app/styles/application.less
+++ b/ambari-web/app/styles/application.less
@@ -2459,10 +2459,6 @@ a:focus {
}
- .add-service-button {
- margin: 20px 20px 10px;
- }
-
.start-stop-all-service-button {
margin: 5px 5px 10px 10px;
text-align: center;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3bac1d4/ambari-web/app/templates/main/service/all_services_actions.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/service/all_services_actions.hbs b/ambari-web/app/templates/main/service/all_services_actions.hbs
index edcd29c..081d3eb 100644
--- a/ambari-web/app/templates/main/service/all_services_actions.hbs
+++ b/ambari-web/app/templates/main/service/all_services_actions.hbs
@@ -24,7 +24,7 @@
</a>
<ul class="pull-left dropdown-menu">
{{#isAuthorized "SERVICE.ADD_DELETE_SERVICES"}}
- <li {{bindAttr class="view.serviceController.isAllServicesInstalled:disabled"}}>
+ <li {{bindAttr class="view.serviceController.isAllServicesInstalled:disabled :add-service-button"}}>
<a href="#"
{{bindAttr class="view.serviceController.isAllServicesInstalled:disabled"}}
{{action gotoAddService target="view.serviceController"}}>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3bac1d4/ambari-web/test/controllers/main/service_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/service_test.js b/ambari-web/test/controllers/main/service_test.js
index 06961a1..440c26e 100644
--- a/ambari-web/test/controllers/main/service_test.js
+++ b/ambari-web/test/controllers/main/service_test.js
@@ -116,18 +116,17 @@ describe('App.MainServiceController', function () {
describe("#isAllServicesInstalled", function() {
beforeEach(function() {
- sinon.stub(App.StackService, 'find').returns([
- Em.Object.create({serviceName: 'S1'})
- ]);
+ this.mock = sinon.stub(App.ServiceSimple, 'find');
});
afterEach(function() {
- App.StackService.find.restore();
+ App.ServiceSimple.find.restore();
});
it("content is null", function() {
mainServiceController.reopen({
'content': null
});
+ this.mock.returns([]);
mainServiceController.propertyDidChange('isAllServicesInstalled');
expect(mainServiceController.get('isAllServicesInstalled')).to.be.false;
});
@@ -136,6 +135,9 @@ describe('App.MainServiceController', function () {
mainServiceController.reopen({
'content': []
});
+ this.mock.returns([
+ {serviceName: 'S1', doNotShowAndInstall: false}
+ ]);
mainServiceController.propertyDidChange('isAllServicesInstalled');
expect(mainServiceController.get('isAllServicesInstalled')).to.be.false;
});
@@ -144,9 +146,23 @@ describe('App.MainServiceController', function () {
mainServiceController.reopen({
'content': [Em.Object.create({serviceName: 'S1'})]
});
+ this.mock.returns([
+ {serviceName: 'S1', doNotShowAndInstall: false}
+ ]);
mainServiceController.propertyDidChange('isAllServicesInstalled');
expect(mainServiceController.get('isAllServicesInstalled')).to.be.true;
});
+ it("content doesn't match stack services", function() {
+ mainServiceController.reopen({
+ 'content': [Em.Object.create({serviceName: 'S1'})]
+ });
+ this.mock.returns([
+ {serviceName: 'S1', doNotShowAndInstall: false},
+ {serviceName: 'S1', doNotShowAndInstall: false}
+ ]);
+ mainServiceController.propertyDidChange('isAllServicesInstalled');
+ expect(mainServiceController.get('isAllServicesInstalled')).to.be.false;
+ });
});
describe('#cluster', function() {
[17/55] [abbrv] ambari git commit: AMBARI-18157: Hive service check
is failing after cluster Kerberization (jluniya)
Posted by ol...@apache.org.
AMBARI-18157: Hive service check is failing after cluster Kerberization (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0e5fb8ee
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0e5fb8ee
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0e5fb8ee
Branch: refs/heads/branch-dev-logsearch
Commit: 0e5fb8ee8e205ccb9da889e44ba77ffbde3f86dd
Parents: fda22e2
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Thu Aug 18 12:01:35 2016 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Thu Aug 18 12:01:35 2016 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 91 ++++++++++++++++++++
.../stacks/HDP/2.1/services/stack_advisor.py | 20 ++++-
.../stacks/2.0.6/common/test_stack_advisor.py | 53 ++++++++++++
3 files changed, 163 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e5fb8ee/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7133c27..7aeda19 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1679,6 +1679,97 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
leafQueueNames.add(leafQueueName)
return leafQueueNames
+ def get_service_component_meta(self, service, component, services):
+ """
+ Function retrieve service component meta information as dict from services.json
+ If no service or component found, would be returned empty dict
+
+ Return value example:
+ "advertise_version" : true,
+ "bulk_commands_display_name" : "",
+ "bulk_commands_master_component_name" : "",
+ "cardinality" : "1+",
+ "component_category" : "CLIENT",
+ "component_name" : "HBASE_CLIENT",
+ "custom_commands" : [ ],
+ "decommission_allowed" : false,
+ "display_name" : "HBase Client",
+ "has_bulk_commands_definition" : false,
+ "is_client" : true,
+ "is_master" : false,
+ "reassign_allowed" : false,
+ "recovery_enabled" : false,
+ "service_name" : "HBASE",
+ "stack_name" : "HDP",
+ "stack_version" : "2.5",
+ "hostnames" : [ "host1", "host2" ]
+
+ :type service str
+ :type component str
+ :type services dict
+ :rtype dict
+ """
+ __stack_services = "StackServices"
+ __stack_service_components = "StackServiceComponents"
+
+ if not services:
+ return {}
+
+ service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+ if len(service_meta) == 0:
+ return {}
+
+ service_meta = service_meta[0]
+ component_meta = [item for item in service_meta["components"] if item[__stack_service_components]["component_name"] == component]
+
+ if len(component_meta) == 0:
+ return {}
+
+ return component_meta[0][__stack_service_components]
+
+ def is_secured_cluster(self, services):
+ """
+ Detects if cluster is secured or not
+ :type services dict
+ :rtype bool
+ """
+ return services and "cluster-env" in services["configurations"] and\
+ "security_enabled" in services["configurations"]["cluster-env"]["properties"] and\
+ services["configurations"]["cluster-env"]["properties"]["security_enabled"].lower() == "true"
+
+ def get_services_list(self, services):
+ """
+ Returns available services as list
+
+ :type services dict
+ :rtype list
+ """
+ if not services:
+ return []
+
+ return [service["StackServices"]["service_name"] for service in services["services"]]
+
+ def get_components_list(self, service, services):
+ """
+ Return list of components for specific service
+ :type service str
+ :type services dict
+ :rtype list
+ """
+ __stack_services = "StackServices"
+ __stack_service_components = "StackServiceComponents"
+
+ if not services:
+ return []
+
+ service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+ if len(service_meta) == 0:
+ return []
+
+ service_meta = service_meta[0]
+ return [item[__stack_service_components]["component_name"] for item in service_meta["components"]]
+
+
def getOldValue(self, services, configType, propertyName):
if services:
if 'changed-configurations' in services.keys():
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e5fb8ee/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 34b4614..e194b3f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -107,7 +107,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
dbConnection = self.getDBConnectionString(hiveEnvProperties['hive_database']).format(hiveServerHost['Hosts']['host_name'], hiveSiteProperties['ambari.hive.db.schema.name'])
putHiveProperty('javax.jdo.option.ConnectionURL', dbConnection)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ servicesList = self.get_services_list(services)
if "PIG" in servicesList:
ambari_user = self.getAmbariUser(services)
ambariHostName = socket.getfqdn()
@@ -120,6 +120,24 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
+ if self.is_secured_cluster(services):
+ appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
+
+ def updateCallback(originalValue, newValue):
+ if originalValue and not originalValue.isspace():
+ hosts = originalValue.split(',')
+
+ if newValue:
+ hosts.extend(newValue)
+
+ result = ','.join(set(hosts))
+ return result
+ else:
+ return newValue
+
+ meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
+ if "hostnames" in meta:
+ appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
putTezProperty = self.putProperty(configurations, "tez-site")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0e5fb8ee/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index e28389e..a70922f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -188,6 +188,59 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
+ def test_get_components_list(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = sorted(["GANGLIA_MONITOR", "GANGLIA_SERVER"])
+
+ services = self.prepareServices(servicesInfo)
+ result = sorted(self.stackAdvisor.get_components_list("GANGLIA", services))
+
+ self.assertEqual(expected, result)
+
+ def test_get_services_list(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = ["GANGLIA"]
+
+ services = self.prepareServices(servicesInfo)
+ result = self.stackAdvisor.get_services_list(services)
+
+ self.assertEqual(expected, result)
+
+ def test_get_service_component_meta(self):
+ servicesInfo = [
+ {
+ "name": "GANGLIA",
+ "components": [
+ {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+ {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+ ]
+ }
+ ]
+ expected = ["host1"]
+
+ services = self.prepareServices(servicesInfo)
+ result = self.stackAdvisor.get_service_component_meta("GANGLIA", "GANGLIA_MONITOR", services)
+
+ self.assertEquals(True, "hostnames" in result)
+ self.assertEqual(expected, result["hostnames"])
+
+
def test_validationWarnMessagesIfLessThanDefault(self):
servicesInfo = [
{
[09/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/python/upgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/upgradeHelper.py b/ambari-server/src/main/python/upgradeHelper.py
new file mode 100644
index 0000000..31aa721
--- /dev/null
+++ b/ambari-server/src/main/python/upgradeHelper.py
@@ -0,0 +1,2338 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+"""
+Upgrade catalog file format description:
+
+
+Format version 1.0
+
+Global section description:
+ STACKNAME - name of stack, for example HDP
+ OLDVERSION - version of stack from which upgrade should be done, used by fromStack script argument
+ NEWVERSION - version of stack to which upgrade should be done, used by toStack script argument
+
+Sub-section options:
+ config-types - contains global per-config settings
+ merged-copy - would merge latest server properties with properties defined in "properties" section,
+ without this option server properties would be rewritten by properties defined in "properties" section
+ required-services - properties from json catalog would be processed only if desired services are present on the cluster
+ property level definition will always override catalog level definition.
+
+Sub-section properties - Contains property definition
+Sub-section property-mapping(optional) - contains mapping of property names in case, if some property changed their name in NEWVERSION
+
+Example:
+
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "STACKNAME",
+ "old-version": "OLDVERSION",
+ "target-version": "NEWVERSION",
+ "options": {
+ "config-types": {
+ "CONFIGTYPE1": {
+ "merged-copy": "yes",
+ "required-services": ["HDFS"]
+ }
+ }
+ },
+ "properties": {
+ "CONFIGTYPE1": {
+ "some_property": "some property value",
+ "some_second_property: {
+ "remove": "yes"
+ },
+ "template_property": {
+ "value": "{TEMPLATE_TAG}",
+ "template": "yes",
+ "required-services": ["HDFS", "YARN"]
+ },
+ "test_property": {
+ "value": "new value",
+ "override: "no", (optional, override already existed property yes/no. Default: yes)
+ "value-required": "old value", (optional, property would be set if the required value is present)
+ "can-create": "no", (optional, process property only if that property present on the server.
+ i.e. ability to create new property. Default: yes)
+ "required-services": ["HDFS", "YARN"], (optional, process property only if selected services existed)
+ "resolve-dependency": "no" (optional, use Stack Advisor to get depended properties changes. Default: no)
+ }
+ }
+ },
+ "property-mapping": {
+ "old-property-name": "new-property-name", (short form, equal to "old-property-name": { "map-to": "new-property-name" })
+ "old-property1-name": { (usually key is an name of the property which need to be mapped, but in case of same
+ property should be set to unique name and "map-from" option used instead)
+ "map-from": "old property name", (optional, define property name which should be mapped)
+ "map-to": "new_property1_name", (optional, new property name. If not set, would be used old property name)
+ "from-catalog": "test", (optional, require "to-catalog. Source of old-property1-name)
+ "to-catalog": "test", (optional, require "from-catalog. Target of new_property1_name)
+ "default": "default value", (optional, if set and old property not exists, new one would be created with default value)
+ "template": "yes", (optional, template parsing for default option)
+ "coerce-to": "pre-defined type", (optional, convert value from one type to another. Types supported:
+ yaml-array - converts string item1,item2 to ['item1', 'item2']
+ )
+ "replace-from": "something", (optional, should be present both from and to. Replace 'from' value to 'to')
+ "replace-to": "something,
+ "required-services": ["YARN"], (optional, process entry if services in the list existed on the cluster
+ }
+ }
+ }
+ ]
+}
+
+More examples available in ambari-server/src/main/resources/upgrade/catalog/
+"""
+
+import getpass
+import optparse
+from pprint import pprint
+import re
+import sys
+import os.path
+import logging
+import time
+import base64
+from urllib2 import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, Request, build_opener, URLError, HTTPError
+
+try:
+ # try to import new simplejson version, which should be faster than outdated python 2.6 version
+ import ambari_simplejson as json
+except ImportError:
+ import json
+
+
+# ==============================
+# Error classes definition
+# ==============================
+class FatalException(Exception):
+ def __init__(self, code, reason):
+ self.code = code
+ self.reason = reason
+
+ def __str__(self):
+ return repr("Fatal exception: %s, exit code %s" % (self.reason, self.code))
+
+ def _get_message(self):
+ return str(self)
+
+
+class ReadOnlyPropertyException(Exception):
+ def __str__(self):
+ return "Property is read-only"
+
+ def _get_message(self):
+ return self.__str__()
+
+
+class NotSupportedCatalogVersion(Exception):
+ def __init__(self, catalog_version):
+ self._version = catalog_version
+
+ def __str__(self):
+ return "Version %s of loaded catalog not supported" % self._version
+
+ def _get_message(self):
+ return self.__str__()
+
+ message = property(__str__)
+
+
+class CatalogNotFoundException(Exception):
+ pass
+
+
+class TemplateProcessingException(Exception):
+ pass
+
+
+class CatalogExistException(Exception):
+ pass
+
+
+class PropertyNotFoundException(Exception):
+ pass
+
+
+class StackNotFoundException(Exception):
+ pass
+
+
+class MalformedPropertyDefinitionException(Exception):
+ pass
+
+
+# ==============================
+# Constant class definition
+# ==============================
+class Const(object):
+ def __new__(cls, *args, **kwargs):
+ raise Exception("Class couldn't be created")
+
+
+class Options(Const):
+ # action commands
+ API_PROTOCOL = "http"
+ API_PORT = "8080"
+
+ GET_MR_MAPPING_ACTION = "save-mr-mapping"
+ VERIFY_ACTION = "verify"
+ DELETE_MR_ACTION = "delete-mr"
+ ADD_YARN_MR2_ACTION = "add-yarn-mr2"
+ MODIFY_CONFIG_ACTION = "update-configs"
+ BACKUP_CONFIG_ACTION = "backup-configs"
+ INSTALL_YARN_MR2_ACTION = "install-yarn-mr2"
+
+ MR_MAPPING_FILE = "mr_mapping"
+ CAPACITY_SCHEDULER_TAG = "capacity-scheduler"
+ REPLACE_JH_HOST_NAME_TAG = "REPLACE_JH_HOST"
+ REPLACE_RM_HOST_NAME_TAG = "REPLACE_RM_HOST"
+ REPLACE_WITH_TAG = "REPLACE_WITH_"
+ PHOENIX_QUERY_SERVER = "PHOENIX_QUERY_SERVER"
+ ZK_OPTIONS = "zoo.cfg"
+ KAFKA_BROKER_CONF = "kafka-broker"
+ RANGER_ADMIN = "admin-properties"
+ RANGER_USERSYNC = "usersync-properties"
+ RANGER_ENV = "ranger-env"
+ KAFKA_PORT = "port"
+ RANGER_EXTERNAL_URL = "policymgr_external_url"
+ ZK_CLIENTPORT = "clientPort"
+ DELETE_OLD_TAG = "DELETE_OLD"
+
+ ZOOKEEPER_SERVER = "ZOOKEEPER_SERVER"
+ KAFKA_BROKER = "KAFKA_BROKER"
+ NAMENODE = "NAMENODE"
+
+ MR_MAPPING = None
+ logger = None
+ server_config_factory = None
+ """:type : ServerConfigFactory"""
+ stack_advisor = None
+ """:type : StackAdvisor"""
+ ambari_server = None
+ """:type : AmbariServer"""
+
+ # Api constants
+ ROOT_URL = None
+ CLUSTER_URL = None
+ COMPONENTS_FORMAT = None
+ TEZ_VIEW_URL = None
+
+ # Curl options
+ CURL_PRINT_ONLY = None
+ CURL_WRITE_ONLY = None
+
+ ARGS = None
+ OPTIONS = None
+ HOST = None
+ CLUSTER_NAME = None
+
+ # for verify action
+ REPORT_FILE = None
+
+ SERVICES = []
+
+ API_TOKENS = {
+ "user": None,
+ "pass": None
+ }
+
+ HEADERS = {
+ 'X-Requested-By': 'upgradeHelper'
+ }
+
+ @classmethod
+ def initialize(cls):
+ cls.ROOT_URL = '%s://%s:%s/api/v1' % (cls.API_PROTOCOL, cls.HOST, cls.API_PORT)
+ cls.CLUSTER_URL = cls.ROOT_URL + "/clusters/%s" % cls.CLUSTER_NAME
+ cls.COMPONENTS_URL = cls.CLUSTER_URL + "/components?fields=ServiceComponentInfo/total_count"
+ cls.COMPONENTS_FORMAT = cls.CLUSTER_URL + "/components/{0}"
+ cls.TEZ_VIEW_URL = cls.ROOT_URL + "/views/TEZ"
+ cls.STACKS_URL = cls.ROOT_URL + "/stacks"
+ cls.STACKS_VERSIONS_URL = cls.STACKS_URL + "/{0}/versions"
+ cls.STACK_ADVISOR_URL = cls.STACKS_VERSIONS_URL + "/{1}/recommendations"
+ cls.AMBARI_SERVER_URL = cls.ROOT_URL + "/services/AMBARI/components/AMBARI_SERVER"
+ cls.AMBARI_AGENTS_URL = cls.ROOT_URL + "/services/AMBARI/components/AMBARI_AGENT"
+ if cls.CLUSTER_NAME is not None and cls.HOST is not None:
+ cls.SERVICES = set(map(lambda x: x.upper(), get_cluster_services()))
+
+ cls.ambari_server = AmbariServer()
+ if not cls.isPropertyAttributesSupported():
+ cls.logger.warning("Property attributes not supported by current Ambari version")
+
+ @classmethod
+ def isPropertyAttributesSupported(cls):
+ if cls.ambari_server.server_version[0] * 10 + cls.ambari_server.server_version[1] >= 17:
+ return True
+ return False
+
+ @classmethod
+ def initialize_logger(cls, filename=None):
+ cls.logger = logging.getLogger('UpgradeHelper')
+ cls.logger.setLevel(logging.DEBUG)
+
+ if filename is not None:
+ handler = logging.FileHandler(filename)
+ handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s - %(message)s'))
+ cls.logger.addHandler(handler)
+ cls.logger.info("")
+ cls.logger.info("Start new logging section")
+
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+ cls.logger.addHandler(handler)
+
+
+class CatConst(Const):
+ VERSION_TAG = "version"
+ STACK_VERSION_OLD = "old-version"
+ STACK_VERSION_TARGET = "target-version"
+ STACK_STAGS_TAG = "stacks"
+ STACK_NAME = "name"
+ CONFIG_OPTIONS = "options"
+ CONFIG_TYPES = "config-types"
+ STACK_PROPERTIES = "properties"
+ STACK_PROPERTIES_ATTRIBUTES = "properties_attributes"
+ PROPERTY_VALUE_TAG = "value"
+ VERSIONS_TAG = "versions"
+ PROPERTY_REMOVE_TAG = "remove"
+ PROPERTY_MAP_TO = "map-to"
+ PROPERTY_MAP_FROM = "map-from"
+ PROPERTY_FROM_CATALOG = "from-catalog"
+ PROPERTY_TO_CATALOG = "to-catalog"
+ PROPERTY_DEFAULT = "default"
+ MERGED_COPY_TAG = "merged-copy"
+ REQUIRED_SERVICES = "required-services"
+ COERCE_TO_PROPERTY_TAG = "coerce-to"
+ RESOLVE_DEPENDENCY_TAG = "resolve-dependency"
+ COERCE_YAML_OPTION_TAG = "yaml-array"
+ REPLACE_FROM_TAG = "replace-from"
+ REPLACE_TO_TAG = "replace-to"
+ OVERRIDE_TAG = "override"
+ ITEMS_TAG = "items"
+ TYPE_TAG = "type"
+ TRUE_TAG = "yes"
+ VALUE_REQUIRED_TAG = "value-required"
+ PROPERTY_CAN_CREATE_TAG = "can-create"
+ STACK_PROPERTIES_MAPPING_LIST_TAG = "property-mapping"
+ VALUE_TEMPLATE_TAG = "template"
+ SEARCH_PATTERN = "(\{[^\{\}]+\})" # {XXXXX}
+ ACTION_COMMIT = "commit"
+ ACTION_RELOAD = "reload"
+ ACTION_RENAME_PROPERTY = "rename-property"
+ TEMPLATE_HANDLER = "template_handler"
+
+
+# ==============================
+# Catalog classes definition
+# ==============================
+
+class AmbariServer(object):
+ def __init__(self):
+ Options.logger.info("Resolving Ambari server configuration ...")
+ self._get_server_info()
+ self._get_agents_info()
+ self._get_components()
+
+ def _get_components(self):
+ info = curl(Options.COMPONENTS_URL, parse=True)
+ self._components = []
+ if CatConst.ITEMS_TAG in info:
+ for item in info[CatConst.ITEMS_TAG]:
+ if "ServiceComponentInfo" in item and "total_count" in item["ServiceComponentInfo"] and \
+ int(item["ServiceComponentInfo"]["total_count"]) > 0 and "component_name" in item["ServiceComponentInfo"]:
+ self._components.append(item["ServiceComponentInfo"]["component_name"])
+
+ def _get_server_info(self):
+ info = curl(Options.AMBARI_SERVER_URL, parse=True)
+ self._server_version = [0, 0, 0]
+
+ if "RootServiceComponents" in info:
+ server_props = info["RootServiceComponents"]
+ ver = server_props["component_version"] if "component_version" in server_props else None
+ try:
+ self._server_version = list(map(lambda x: int(x), ver.split(".")))
+ except ValueError:
+ pass
+
+ def _get_agents_info(self):
+ info = curl(Options.AMBARI_AGENTS_URL, parse=True)
+ self._agents = []
+ if "hostComponents" in info:
+ agent_props = info["hostComponents"]
+ self._agents = list(map(lambda x: x["RootServiceHostComponents"]["host_name"], agent_props))
+
+ @property
+ def components(self):
+ return self._components
+
+ @property
+ def server_version(self):
+ return self._server_version
+
+ @property
+ def agent_hosts(self):
+ return self._agents
+
+class StackAdvisorFactory(object):
+ def __init__(self):
+ self._stack_info = self._load_stack_info()
+
+ def _load_stack_versions(self, stack):
+ versions = curl(Options.STACKS_VERSIONS_URL.format(stack), parse=True)
+ if CatConst.ITEMS_TAG in versions:
+ versions = list(map(lambda x: x["Versions"]["stack_version"], versions[CatConst.ITEMS_TAG]))
+
+ return versions
+
+ def _load_stack_info(self):
+ stacks = curl(Options.STACKS_URL, parse=True)
+ if CatConst.ITEMS_TAG in stacks:
+ stacks = list(map(lambda x: x["Stacks"]["stack_name"], stacks["items"]))
+ else:
+ stacks = {}
+
+ stacks_dict = {}
+
+ for stack in stacks:
+ stacks_dict[stack] = self._load_stack_versions(stack)
+
+ return stacks_dict
+
+ def get_instance(self, stack, version):
+ sversion = Options.ambari_server.server_version
+ if sversion[0] * 10 + sversion[1] < 21:
+ Options.logger.warning("Ambari server version \"%s.%s.%s\" doesn't support property dependencies suggestion" %
+ (sversion[0], sversion[1], sversion[2]))
+ return BaseStackAdvisor(stack, version)
+
+ if stack in self._stack_info and version in self._stack_info[stack]:
+ return StackAdvisor(stack, version)
+ else:
+ raise StackNotFoundException("Stack %s-%s not exist on the server" % (stack, version))
+
+class StackAdvisorRequestProperty(object):
+ def __init__(self, catalog, property_name):
+ self._catalog = catalog
+ self._property_name = property_name
+
+ @property
+ def catalog(self):
+ return self._catalog
+
+ @property
+ def name(self):
+ return self._property_name
+
+ def get_json(self):
+ return {
+ "type": self.catalog,
+ "name": self.name
+ }
+
+
+class BaseStackAdvisor(object):
+ def __init__(self, stack, version):
+ self._req_url = Options.STACK_ADVISOR_URL.format(stack, version)
+
+ def get_suggestion(self, cfg_factory, changed_properties):
+ return {}
+
+
+class StackAdvisor(BaseStackAdvisor):
+ def __init__(self, stack, version):
+ super(StackAdvisor, self).__init__(stack, version)
+
+ def _transform_properties(self, cfg_factory):
+ """
+ Transform properties list to blueprint output format
+ :type cfg_factory: ServerConfigFactory
+ :rtype dict
+ """
+ props = cfg_factory.get_json()
+ for cfg in props:
+ props[cfg] = {
+ "properties": props[cfg]
+ }
+
+ return props
+
+ def _from_blueprint_properties_transform(self, props):
+ """
+ Transform SA response to dict
+ """
+ for p in props:
+ rprop = {}
+ if "properties" in props[p] and props[p]["properties"] is not None:
+ rprop = props[p]["properties"]
+ if "property_attributes" in props[p]:
+ for property_attribute in props[p]["property_attributes"]:
+ if "delete" in props[p]["property_attributes"][property_attribute] and \
+ props[p]["property_attributes"][property_attribute]["delete"] == "true":
+ rprop[property_attribute] = None
+
+ props[p] = rprop
+
+ return props
+
+ def _generate_req_properties(self, properties):
+ rlist = []
+ for item in properties:
+ if isinstance(item, StackAdvisorRequestProperty):
+ rlist.append(item.get_json())
+ return rlist
+
+ def get_suggestion(self, cfg_factory, changed_properties):
+ """
+ :type cfg_factory: ServerConfigFactory
+ :type catalog_name str
+ :type changed_properties: list
+ :rtype dict
+ """
+ request = {
+ "recommend": "configuration-dependencies",
+ "hosts": Options.ambari_server.agent_hosts,
+ "services": list(Options.SERVICES),
+ "changed_configurations": self._generate_req_properties(changed_properties),
+ "recommendations": {
+ "blueprint": {
+ "host_groups": [],
+ "configurations": self._transform_properties(cfg_factory),
+ "blueprint_cluster_binding": {}
+ }
+ }
+ }
+ response = curl(self._req_url, request_type="POST", data=request, parse=True)
+ if "resources" in response and isinstance(response["resources"], list) and len(response["resources"]) > 0:
+ response = response["resources"][0]
+ if "recommendations" in response and "blueprint" in response["recommendations"] and \
+ "configurations" in response["recommendations"]["blueprint"]:
+ return self._from_blueprint_properties_transform(response["recommendations"]["blueprint"]["configurations"])
+
+ return {}
+
+
+
+class UpgradeCatalogFactory(object):
+ # versions of catalog which is currently supported
+ _supported_catalog_versions = ["1.0"]
+
+ # private variables
+ _json_catalog = None
+
+ def __init__(self, path):
+ self._load(path)
+
+ def _load(self, path):
+ f = None
+ try:
+ f = open(path, 'r')
+ json_string = f.read()
+ self._json_catalog = json.loads(json_string)
+ self._parse_upgrade_catalog()
+ except IOError as e:
+ raise FatalException(e.errno, "Couldn't open upgrade catalog file %s: %s" % (path, e.strerror))
+ except NotSupportedCatalogVersion as e:
+ raise FatalException(1, e.message)
+ except ValueError as e:
+ raise FatalException(1, "Malformed upgrade catalog: %s" % e.message)
+ finally:
+ try:
+ if f is not None:
+ f.close()
+ except IOError as e:
+ pass
+
+ def _parse_upgrade_catalog(self):
+ catalog_version = None
+ if CatConst.VERSION_TAG in self._json_catalog:
+ catalog_version = self._json_catalog[CatConst.VERSION_TAG]
+
+ if catalog_version is None or catalog_version not in self._supported_catalog_versions:
+ raise NotSupportedCatalogVersion(str(catalog_version))
+
+ def get_catalog(self, from_version=None, to_version=None):
+ search_version = {
+ CatConst.STACK_VERSION_OLD: from_version,
+ CatConst.STACK_VERSION_TARGET: to_version
+ }
+
+ for stack in self._json_catalog[CatConst.STACK_STAGS_TAG]:
+ version = {
+ CatConst.STACK_VERSION_OLD: stack[CatConst.STACK_VERSION_OLD],
+ CatConst.STACK_VERSION_TARGET: stack[CatConst.STACK_VERSION_TARGET]
+ }
+ if version == search_version:
+ return UpgradeCatalog(catalog=stack, version=version)
+
+ return None
+
+
+class UpgradeCatalog(object):
+ # private variables
+ _json_catalog = None
+ _properties_catalog = None
+ _properties_map_catalog = None
+ _version = None
+ _search_pattern = None
+ _catalog_options = None
+
+ def __init__(self, catalog=None, version=None):
+ self._handlers = {}
+ self._json_catalog = catalog
+ self._version = version
+ self._search_pattern = re.compile(CatConst.SEARCH_PATTERN)
+
+ if CatConst.STACK_PROPERTIES in catalog:
+ self._properties_catalog = self._format_catalog_properties(catalog[CatConst.STACK_PROPERTIES])
+
+ if CatConst.STACK_PROPERTIES_MAPPING_LIST_TAG in catalog:
+ self._properties_map_catalog = PropertyMapping(catalog[CatConst.STACK_PROPERTIES_MAPPING_LIST_TAG])
+ else:
+ self._properties_map_catalog = PropertyMapping()
+
+ if catalog is not None and CatConst.CONFIG_OPTIONS in catalog \
+ and CatConst.CONFIG_TYPES in catalog[CatConst.CONFIG_OPTIONS]:
+ self._catalog_options = catalog[CatConst.CONFIG_OPTIONS]
+
+ def add_handler(self, name, handler):
+ if name not in self._handlers:
+ self._handlers[name] = handler
+
+ def _format_catalog_properties(self, properties):
+ """
+ Transform properties from short form to normal one:
+ "property": "text" => "property": { "value": "text" }
+ :param properties: dict
+ :return: dict
+ """
+ for config_item in properties:
+ cfg_item = properties[config_item]
+
+ """
+ case when "properties": {
+ "yarn-site": {
+ .....
+ }
+ }
+ is set like "properties": {
+ "yarn-site": ""
+ }
+ """
+ if not isinstance(cfg_item, dict):
+ raise MalformedPropertyDefinitionException("The property catalog '%s' definition error" % config_item)
+
+ properties[config_item] = dict(zip(
+ cfg_item.keys(),
+ map(lambda x: x if isinstance(x, dict) or isinstance(x, list) else {CatConst.PROPERTY_VALUE_TAG: x}, cfg_item.values())
+ ))
+ return properties
+
+ @property
+ def version(self):
+ return "%s-%s" % (self._version[CatConst.STACK_VERSION_OLD], self._version[CatConst.STACK_VERSION_TARGET])
+
+ @property
+ def target_version(self):
+ return self._version[CatConst.STACK_VERSION_TARGET]
+
+ @property
+ def source_version(self):
+ return self._version[CatConst.STACK_VERSION_OLD]
+
+ def get_parsed_version(self):
+ """
+ Get numeric representation of the version for comparation purposes
+
+ Example:
+ 1.3-2.1 will be represented as { from: 13, to: 21 }
+
+ :return: Numeric version
+ """
+ v_from = self._version[CatConst.STACK_VERSION_OLD].split(".")
+ v_to = self._version[CatConst.STACK_VERSION_TARGET].split(".")
+ try:
+ v_from = int(v_from[0]) * 10 + int(v_from[1])
+ v_to = int(v_to[0]) * 10 + int(v_to[1])
+ except ValueError:
+ v_from = 0
+ v_to = 0
+
+ version = {
+ "from": v_from,
+ "to": v_to
+ }
+
+ return version
+
+ @property
+ def name(self):
+ if CatConst.STACK_NAME in self._json_catalog:
+ return self._json_catalog[CatConst.STACK_NAME]
+ return ""
+
+ @property
+ def mapping(self):
+ return self._properties_map_catalog
+
+ @property
+ def items(self):
+ return self._properties_catalog
+
+ @property
+ def options(self):
+ if CatConst.CONFIG_TYPES in self._catalog_options:
+ return self._catalog_options[CatConst.CONFIG_TYPES]
+ return {}
+
+ @property
+ def action_handlers(self):
+ return self._handlers
+
+ @property
+ def tag_search_pattern(self):
+ return self._search_pattern
+
+ def __handle_remove_tag(self, name, catalog_item_name, catalog_property_item, properties):
+ """
+ :type name str
+ :type catalog_item_name str
+ :type catalog_property_item dict
+ :type properties dict
+ """
+ if CatConst.PROPERTY_REMOVE_TAG in catalog_property_item and \
+ catalog_property_item[CatConst.PROPERTY_REMOVE_TAG] == CatConst.TRUE_TAG and \
+ catalog_item_name in properties:
+ del properties[catalog_item_name]
+
+ def __handle_template_tag_sub(self, catalog_item_name, catalog_property_item):
+ """
+ :type catalog_item_name str
+ :type catalog_property_item dict
+ """
+ if CatConst.TEMPLATE_HANDLER in self._handlers and self._handlers is not None and \
+ CatConst.VALUE_TEMPLATE_TAG in catalog_property_item and catalog_property_item[
+ CatConst.VALUE_TEMPLATE_TAG] == CatConst.TRUE_TAG:
+ try:
+ parsed_value = self._handlers[CatConst.TEMPLATE_HANDLER](
+ self,
+ self._search_pattern.findall(catalog_property_item[CatConst.PROPERTY_VALUE_TAG]),
+ catalog_property_item[CatConst.PROPERTY_VALUE_TAG]
+ )
+ catalog_property_item[CatConst.PROPERTY_VALUE_TAG] = parsed_value
+ except TemplateProcessingException:
+ pass
+
+ def __handle_add_new(self, name, catalog_item_name, catalog_property_item, properties):
+ """
+ :type name str
+ :type catalog_item_name str
+ :type catalog_property_item dict
+ :type properties dict
+ """
+ catalog_property_item = dict(catalog_property_item)
+ can_add_new = not (CatConst.PROPERTY_CAN_CREATE_TAG in catalog_property_item and
+ catalog_property_item[CatConst.PROPERTY_CAN_CREATE_TAG].upper() == "NO")
+ if CatConst.PROPERTY_VALUE_TAG in catalog_property_item and catalog_item_name not in properties and can_add_new:
+ self.__handle_template_tag_sub(catalog_item_name, catalog_property_item)
+ properties[catalog_item_name] = catalog_property_item[CatConst.PROPERTY_VALUE_TAG]
+
+ def __handle_change_existing(self, name, catalog_item_name, catalog_property_item, properties):
+ """
+ :type name str
+ :type catalog_item_name str
+ :type catalog_property_item dict
+ :type properties dict
+ """
+ catalog_property_item = dict(catalog_property_item)
+ can_override = True
+
+ if CatConst.OVERRIDE_TAG in catalog_property_item and catalog_property_item[CatConst.OVERRIDE_TAG] != CatConst.TRUE_TAG:
+ can_override = False
+
+ if CatConst.PROPERTY_VALUE_TAG in catalog_property_item and catalog_item_name in properties and can_override:
+ self.__handle_template_tag_sub(catalog_item_name, catalog_property_item)
+ properties[catalog_item_name] = catalog_property_item[CatConst.PROPERTY_VALUE_TAG]
+ return properties
+
+ def __handle_dependency_tag(self, name, catalog_item_name, catalog_property_item, properties):
+ """
+ :type name str
+ :type catalog_item_name str
+ :type catalog_property_item dict
+ :type properties dict
+ """
+ if CatConst.RESOLVE_DEPENDENCY_TAG in catalog_property_item and \
+ catalog_property_item[CatConst.RESOLVE_DEPENDENCY_TAG] == CatConst.TRUE_TAG:
+ sa_suggestions = Options.stack_advisor.get_suggestion(Options.server_config_factory,
+ [StackAdvisorRequestProperty(name, catalog_item_name)])
+ for sa_catalog in sa_suggestions:
+ # create new config group if not existed
+ if sa_catalog not in Options.server_config_factory.items():
+ Options.server_config_factory.create_config(sa_catalog)
+
+ catalog_properties = Options.server_config_factory.get_config(sa_catalog).properties
+ for sa_property in sa_suggestions[sa_catalog]:
+ if sa_suggestions[sa_catalog][sa_property] is None and sa_property in catalog_properties:
+ print "rem %s:%s" % (sa_catalog, sa_property)
+ del catalog_properties[sa_property]
+ elif sa_suggestions[sa_catalog][sa_property] is not None:
+ catalog_properties[sa_property] = sa_suggestions[sa_catalog][sa_property]
+
+
+ def __can_handler_execute(self, catalog_options, catalog_property_item, property_item, properties):
+ """
+ :type catalog_options dict
+ :type catalog_property_item str
+ :type property_item dict
+ :type properties dict
+ """
+ can_process = True
+
+ # process required services tag
+ required_list = None
+
+ if CatConst.REQUIRED_SERVICES in catalog_options and catalog_options[CatConst.REQUIRED_SERVICES] is not None and \
+ isinstance(catalog_options[CatConst.REQUIRED_SERVICES], list):
+ required_list = catalog_options[CatConst.REQUIRED_SERVICES]
+
+ if CatConst.REQUIRED_SERVICES in property_item and property_item[CatConst.REQUIRED_SERVICES] is not None and\
+ isinstance(property_item[CatConst.REQUIRED_SERVICES], list):
+ required_list = property_item[CatConst.REQUIRED_SERVICES]
+
+ if required_list is not None:
+ can_process = can_process and is_services_exists(required_list)
+
+ if CatConst.VALUE_REQUIRED_TAG in property_item and property_item[CatConst.VALUE_REQUIRED_TAG] is not None and\
+ CatConst.PROPERTY_VALUE_TAG in property_item and catalog_property_item in properties:
+ can_process = properties[catalog_property_item] == property_item[CatConst.VALUE_REQUIRED_TAG]
+
+ return can_process
+
+ def process_simple_transformations(self, name, properties):
+ """
+ :type properties dict
+ :type name str
+ """
+ tag_handlers = [
+ self.__handle_add_new,
+ self.__handle_change_existing,
+ self.__handle_dependency_tag,
+ self.__handle_remove_tag
+ ]
+ # catalog has no update entries for this config group
+ if name not in self._properties_catalog:
+ return 0
+
+ catalog_item = self._properties_catalog[name]
+ for catalog_property_item in catalog_item.keys():
+ catalog_options = self.options[name] if name in self.options else {}
+ if self.__can_handler_execute(catalog_options, catalog_property_item, catalog_item[catalog_property_item], properties):
+ for handler in tag_handlers:
+ handler(name, catalog_property_item, catalog_item[catalog_property_item], properties)
+
+
+class PropertyMapping(object):
+ _mapping_list = {}
+
+ def __init__(self, map_list=None):
+ if map_list is not None:
+ self._mapping_list = self._convert_list(map_list)
+
+ def _convert_list(self, map_list):
+ return dict(zip(
+ map_list.keys(),
+ map(lambda x: x if isinstance(x, dict) else {CatConst.PROPERTY_MAP_TO: x}, map_list.values())
+ ))
+
+ def get(self, old_property_name):
+ """
+ Get property mapping dict
+ :old_property_name str
+ :return dict
+ """
+ if old_property_name in self._mapping_list:
+ return self._mapping_list[old_property_name]
+
+ raise PropertyNotFoundException("Property %s from property mapping section not found" % old_property_name)
+
+ def list(self):
+ return self._mapping_list.keys()
+
+ def get_mapped_name(self, old_property_name):
+ if CatConst.PROPERTY_MAP_TO not in self.get(old_property_name):
+ raise MalformedPropertyDefinitionException("%s option is not set for %s property" %
+ (CatConst.PROPERTY_MAP_TO, old_property_name))
+ return self.get(old_property_name)[CatConst.PROPERTY_MAP_TO]
+
+ def exists(self, old_property_name):
+ return old_property_name in self._mapping_list
+
+
+class ServerConfigFactory(object):
+ def __init__(self):
+ self.__observers = []
+ self._server_catalogs = {}
+ self._load_configs()
+
+ def subscribe(self, name, config_item):
+ self.__observers.append((name, config_item))
+
+ def _load_configs(self):
+ Options.logger.info('Getting latest cluster configuration from the server...')
+ new_configs = get_config_resp_all()
+ for config_item in new_configs:
+ if config_item in self._server_catalogs:
+ self.notify_observer(config_item, CatConst.ACTION_RELOAD, new_configs[config_item])
+ else:
+ self._server_catalogs[config_item] = ServerConfig(self, config_item, new_configs[config_item])
+
+ def notify_observers(self, action, arg=None):
+ for name, config_item in self.__observers:
+ if config_item is not None and name in self._server_catalogs:
+ config_item.notify(action, arg)
+
+ def notify_observer(self, _name, action, arg=None):
+ for name, config_item in self.__observers:
+ if config_item is not None and name == _name and name in self._server_catalogs:
+ config_item.notify(action, arg)
+
+ def __str__(self):
+ catalogs = {}
+ for cfg in self._server_catalogs:
+ catalogs[cfg] = str(self._server_catalogs[cfg])
+
+ return json.dumps(catalogs)
+
+ def get_json(self):
+ catalogs = {}
+ for cfg in self._server_catalogs:
+ catalogs[cfg] = self._server_catalogs[cfg].properties
+
+ return catalogs
+ def get_config(self, name):
+ """
+ Get configuration item object
+ :type name str
+ :rtype: ServerConfig
+ """
+ if name in self._server_catalogs:
+ return self._server_catalogs[name]
+
+ raise CatalogNotFoundException("Server catalog item \"%s\" not found" % name)
+
+ def create_config(self, name):
+ if name not in self._server_catalogs:
+ self._server_catalogs[name] = ServerConfig(self, name, {CatConst.STACK_PROPERTIES: {}})
+ else:
+ raise CatalogExistException("Config group \"%s\" already existed" % name)
+
+ def items(self):
+ return self._server_catalogs.keys()
+
+ def reload(self):
+ self._load_configs()
+
+ def process_mapping_transformations(self, catalog):
+ """
+ :type catalog UpgradeCatalog
+ """
+ for map_item in catalog.mapping.list():
+ self._process_single_map_transformation(catalog, map_item, catalog.mapping.get(map_item))
+
+ def _process_default_template_map_replacement(self, catalog, item):
+ """
+ :type catalog: UpgradeCatalog
+ :type item: dict
+ """
+ if CatConst.VALUE_TEMPLATE_TAG in item and CatConst.TEMPLATE_HANDLER in catalog.action_handlers and\
+ CatConst.PROPERTY_DEFAULT in item and item[CatConst.VALUE_TEMPLATE_TAG] == CatConst.TRUE_TAG:
+ try:
+ parsed_value = catalog.action_handlers[CatConst.TEMPLATE_HANDLER](
+ catalog,
+ catalog.tag_search_pattern.findall(item[CatConst.PROPERTY_DEFAULT]),
+ item[CatConst.PROPERTY_DEFAULT]
+ )
+ item[CatConst.PROPERTY_DEFAULT] = parsed_value
+ except TemplateProcessingException:
+ pass
+
+ def _process_property_value_transformation(self, catalog, property_map_definition, old_value):
+ """
+ :type catalog: UpgradeCatalog
+ :type property_map_definition: dict
+ :type old_value: str
+ :return: str
+ """
+
+ tmp = old_value
+
+ if CatConst.REPLACE_FROM_TAG in property_map_definition and CatConst.REPLACE_TO_TAG in property_map_definition and\
+ property_map_definition[CatConst.REPLACE_TO_TAG] is not None and property_map_definition[CatConst.REPLACE_FROM_TAG] is not None:
+ tmp = tmp.replace(property_map_definition[CatConst.REPLACE_FROM_TAG], property_map_definition[CatConst.REPLACE_TO_TAG])
+
+ if CatConst.COERCE_TO_PROPERTY_TAG in property_map_definition:
+ if property_map_definition[CatConst.COERCE_TO_PROPERTY_TAG] == CatConst.COERCE_YAML_OPTION_TAG:
+ # for example c6401,c6402 into ['c6401','c6402']
+ data = list(map(lambda x: "'%s'" % x.strip(), tmp.split(',')))
+ tmp = "[%s]" % ','.join(data)
+
+ return tmp
+
+ def _process_single_map_transformation(self, catalog, map_item_name, map_property_item):
+ """
+ :type catalog UpgradeCatalog
+ :type map_item_name str
+ :type map_property_item dict
+ """
+ old_property_name = map_item_name
+
+ # map-from item name could be re-defined via PROPERTY_MAP_FROM property to avoid duplicate entries
+ if CatConst.PROPERTY_MAP_FROM in map_property_item and map_property_item[CatConst.PROPERTY_MAP_FROM] is not None:
+ old_property_name = map_property_item[CatConst.PROPERTY_MAP_FROM]
+
+ new_property_name = old_property_name
+
+ if CatConst.PROPERTY_MAP_TO in map_property_item:
+ new_property_name = map_property_item[CatConst.PROPERTY_MAP_TO]
+
+ # process first required section
+ required_services = map_property_item[CatConst.REQUIRED_SERVICES] if CatConst.REQUIRED_SERVICES in map_property_item else None
+
+ # process required-services tag
+ if required_services is not None and not is_services_exists(required_services):
+ return 0
+
+ # process template tag
+ self._process_default_template_map_replacement(catalog, map_property_item)
+
+ source_cfg_group = map_property_item[CatConst.PROPERTY_FROM_CATALOG] if CatConst.PROPERTY_FROM_CATALOG in map_property_item and\
+ map_property_item[CatConst.PROPERTY_FROM_CATALOG] != "" else None
+ target_cfg_group = map_property_item[CatConst.PROPERTY_TO_CATALOG] if CatConst.PROPERTY_TO_CATALOG in map_property_item and \
+ map_property_item[CatConst.PROPERTY_TO_CATALOG] != ""else None
+ default_value = map_property_item[CatConst.PROPERTY_DEFAULT] if CatConst.PROPERTY_DEFAULT in map_property_item and \
+ map_property_item[CatConst.PROPERTY_DEFAULT] != "" else None
+
+ if source_cfg_group is None and target_cfg_group is None: # global scope mapping renaming
+ self.notify_observers(CatConst.ACTION_RENAME_PROPERTY, [old_property_name, new_property_name,
+ self._process_property_value_transformation,
+ catalog,
+ map_property_item
+ ])
+ elif source_cfg_group is not None and target_cfg_group is not None: # group-to-group moving
+ if source_cfg_group in self._server_catalogs and target_cfg_group in self._server_catalogs:
+ old_cfg_group = self.get_config(source_cfg_group).properties
+ new_cfg_group = self.get_config(target_cfg_group).properties
+
+ if old_property_name in old_cfg_group:
+ new_cfg_group[new_property_name] = self._process_property_value_transformation(catalog, map_property_item, old_cfg_group[old_property_name])
+ if new_property_name != old_property_name:
+ del old_cfg_group[old_property_name]
+ elif old_property_name not in old_cfg_group and default_value is not None:
+ new_cfg_group[new_property_name] = default_value
+
+ def commit(self):
+ self.notify_observers(CatConst.ACTION_COMMIT)
+
+
+class ServerConfig(object):
+ def __init__(self, factory, name, initial_configs):
+ """
+ Initialize configuration item
+ :factory ServerConfigFactory
+ """
+ factory.subscribe(name, self)
+ self._configs = initial_configs
+ self._hash = self._calculate_hash()
+ self._name = name
+
+ def _calculate_hash(self):
+ return hash(str(self._configs))
+
+ def notify(self, action, arg=None):
+ if action == CatConst.ACTION_RELOAD:
+ self._configs = arg
+ self._hash = self._calculate_hash()
+ elif action == CatConst.ACTION_COMMIT:
+ self._commit()
+ elif action == CatConst.ACTION_RENAME_PROPERTY and isinstance(arg, list) and len(arg) == 5:
+ self._rename_property(*arg)
+
+ def _rename_property(self, old_name, new_name, transform_func, catalog, map_item):
+ """
+ :type old_name: str
+ :type new_name: str
+ :type transform_func: function
+ :type catalog: UpgradeCatalog
+ :type map_item: dict
+ :return:
+ """
+ if old_name in self.properties:
+ old_property_value = self.properties[old_name]
+ if transform_func is not None:
+ self.properties[new_name] = transform_func(catalog, map_item, old_property_value)
+ else:
+ self.properties[new_name] = old_property_value
+
+ if old_name != new_name:
+ del self.properties[old_name]
+
+ def is_attributes_exists(self):
+ return CatConst.STACK_PROPERTIES_ATTRIBUTES in self._configs
+
+ def __str__(self):
+ return json.dumps(self.properties)
+
+ @property
+ def properties(self):
+ return self._configs[CatConst.STACK_PROPERTIES]
+
+ @properties.setter
+ def properties(self, value):
+ self._configs[CatConst.STACK_PROPERTIES] = value
+
+ @property
+ def attributes(self):
+ return self._configs[CatConst.STACK_PROPERTIES_ATTRIBUTES]
+
+ @attributes.setter
+ def attributes(self, value):
+ self._configs[CatConst.STACK_PROPERTIES_ATTRIBUTES] = value
+
+ def _commit(self):
+ if self._hash != self._calculate_hash():
+ Options.logger.info("Committing changes for \"%s\" configuration group ..." % self._name)
+ if self.is_attributes_exists():
+ update_config(self.properties, self._name, self.attributes)
+ else:
+ update_config(self.properties, self._name)
+
+ def clear(self):
+ self.properties = {}
+ self.attributes = {}
+
+ def merge(self, catalog_item):
+ """
+ :type catalog_item UpgradeCatalog
+ """
+ # handle "merged-copy" tag
+ config_options = catalog_item.options[self._name] if self._name in catalog_item.options else {}
+ clear_properties = not (CatConst.MERGED_COPY_TAG in config_options and
+ config_options[CatConst.MERGED_COPY_TAG] == CatConst.TRUE_TAG)
+ if clear_properties:
+ self.clear()
+ Options.logger.info("Processing configuration group: %s", self._name)
+ catalog_item.process_simple_transformations(self._name, self.properties)
+
+
+def write_mapping(hostmapping):
+ if os.path.isfile(Options.MR_MAPPING_FILE):
+ os.remove(Options.MR_MAPPING_FILE)
+ json.dump(hostmapping, open(Options.MR_MAPPING_FILE, 'w'))
+
+
+def read_mapping():
+ if os.path.isfile(Options.MR_MAPPING_FILE):
+ if Options.MR_MAPPING is not None:
+ return Options.MR_MAPPING
+ else:
+ Options.MR_MAPPING = json.load(open(Options.MR_MAPPING_FILE))
+ return Options.MR_MAPPING
+ else:
+ raise FatalException(-1, "MAPREDUCE host mapping file, mr_mapping, is not available or badly formatted. Execute "
+ "action save-mr-mapping. Ensure the file is present in the directory where you are "
+ "executing this command.")
+
+
+def get_mr1_mapping():
+ components = ["MAPREDUCE_CLIENT", "JOBTRACKER", "TASKTRACKER", "HISTORYSERVER"]
+ GET_URL_FORMAT = Options.CLUSTER_URL + '/services/MAPREDUCE/components/%s'
+ hostmapping = {}
+ for component in components:
+ hostlist = []
+ structured_resp = curl(GET_URL_FORMAT % component, parse=True, validate=True)
+
+ if 'host_components' in structured_resp:
+ for hostcomponent in structured_resp['host_components']:
+ if 'HostRoles' in hostcomponent:
+ if 'host_name' in hostcomponent['HostRoles']:
+ hostlist.append(hostcomponent['HostRoles']['host_name'])
+
+ hostmapping[component] = hostlist
+ write_mapping(hostmapping)
+
+ pprint("File mr_mapping contains the host mapping for mapreduce components. This file is critical for later "
+ "steps.")
+
+
+def get_YN_input(prompt, default):
+ yes = set(['yes', 'ye', 'y'])
+ no = set(['no', 'n'])
+ return get_choice_string_input(prompt, default, yes, no)
+
+
+def get_choice_string_input(prompt, default, firstChoice, secondChoice):
+ choice = raw_input(prompt).lower()
+ if choice in firstChoice:
+ return True
+ elif choice in secondChoice:
+ return False
+ elif choice is "": # Just enter pressed
+ return default
+ else:
+ print "input not recognized, please try again: "
+ return get_choice_string_input(prompt, default, firstChoice, secondChoice)
+
+
+def delete_mr():
+ saved_mr_mapping = get_YN_input("Have you saved MR host mapping using action save-mr-mapping [y/n] (n)? ", False)
+ if not saved_mr_mapping:
+ raise FatalException(1, "Ensure MAPREDUCE host component mapping is saved before deleting it. Use action "
+ "save-mr-mapping.")
+
+ SERVICE_URL_FORMAT = Options.CLUSTER_URL + '/services/MAPREDUCE'
+ COMPONENT_URL_FORMAT = Options.CLUSTER_URL + '/hosts/%s/host_components/%s'
+ NON_CLIENTS = ["JOBTRACKER", "TASKTRACKER", "HISTORYSERVER"]
+ PUT_IN_DISABLED = {
+ "HostRoles": {
+ "state": "DISABLED"
+ }
+ }
+
+ hostmapping = read_mapping()
+
+ for key, value in hostmapping.items():
+ if (key in NON_CLIENTS) and (len(value) > 0):
+ for host in value:
+ curl(COMPONENT_URL_FORMAT % (host, key), request_type="PUT", data=PUT_IN_DISABLED,
+ validate=True)
+
+ curl(SERVICE_URL_FORMAT, request_type="DELETE", validate=True)
+
+
+def get_cluster_stackname():
+ VERSION_URL_FORMAT = Options.CLUSTER_URL + '?fields=Clusters/version'
+
+ structured_resp = curl(VERSION_URL_FORMAT, validate=True, parse=True)
+
+ if 'Clusters' in structured_resp:
+ if 'version' in structured_resp['Clusters']:
+ return structured_resp['Clusters']['version']
+
+ raise FatalException(-1, "Unable to get the cluster version")
+
+
+def has_component_in_stack_def(stack_name, service_name, component_name):
+ STACK_COMPONENT_URL_FORMAT = Options.ROOT_URL + '/stacks2/{0}/versions/{1}/stackServices/{2}/serviceComponents/{3}'
+ stack, stack_version = stack_name.split('-')
+
+ try:
+ curl(STACK_COMPONENT_URL_FORMAT.format(stack, stack_version, service_name, component_name),
+ validate=True)
+ return True
+ except FatalException:
+ return False
+
+
+def add_services():
+ SERVICE_URL_FORMAT = Options.CLUSTER_URL + '/services/{0}'
+ COMPONENT_URL_FORMAT = SERVICE_URL_FORMAT + '/components/{1}'
+ HOST_COMPONENT_URL_FORMAT = Options.CLUSTER_URL + '/hosts/{0}/host_components/{1}'
+ service_comp = {
+ "YARN": ["NODEMANAGER", "RESOURCEMANAGER", "YARN_CLIENT"],
+ "MAPREDUCE2": ["HISTORYSERVER", "MAPREDUCE2_CLIENT"]}
+ new_old_host_map = {
+ "NODEMANAGER": "TASKTRACKER",
+ "HISTORYSERVER": "HISTORYSERVER",
+ "RESOURCEMANAGER": "JOBTRACKER",
+ "YARN_CLIENT": "MAPREDUCE_CLIENT",
+ "MAPREDUCE2_CLIENT": "MAPREDUCE_CLIENT"}
+
+ stack_name = get_cluster_stackname()
+ stack_has_ats = has_component_in_stack_def(stack_name, "YARN", "APP_TIMELINE_SERVER")
+
+ # if upgrading to stack > 2.1 (which has ats)
+ if stack_has_ats:
+ service_comp["YARN"].append("APP_TIMELINE_SERVER")
+ new_old_host_map["APP_TIMELINE_SERVER"] = "JOBTRACKER"
+
+ hostmapping = read_mapping()
+
+ for service in service_comp.keys():
+ curl(SERVICE_URL_FORMAT.format(service), validate=True, request_type="POST")
+
+ for component in service_comp[service]:
+ curl(COMPONENT_URL_FORMAT.format(service, component),
+ validate=True, request_type="POST")
+
+ for host in hostmapping[new_old_host_map[component]]:
+ curl(HOST_COMPONENT_URL_FORMAT.format(host, component),
+ validate=True, request_type="POST")
+
+
+def update_config(properties, config_type, attributes=None):
+ tag = "version" + str(int(time.time() * 1000))
+ properties_payload = {"Clusters": {"desired_config": {"type": config_type, "tag": tag, "properties": properties}}}
+ if attributes is not None:
+ properties_payload["Clusters"]["desired_config"]["properties_attributes"] = attributes
+
+ expect_body = config_type != "cluster-env" # ToDo: make exceptions more flexible
+
+ curl(Options.CLUSTER_URL, request_type="PUT", data=properties_payload, validate=True, soft_validation=True)
+
+
+def build_all_options(desired_configs):
+ """
+ Get all configs in the old-fashion way ( versions below 1.7.0 doesn't support "properties" filter )
+ """
+ config_url_tpl = Options.CLUSTER_URL + "/configurations?type={0}&tag={1}"
+ all_options = {CatConst.ITEMS_TAG: []}
+ for config in desired_configs:
+ cfg_item = curl(config_url_tpl.format(config, desired_configs[config]["tag"]), parse=True, validate=True)
+ if CatConst.ITEMS_TAG in cfg_item and len(cfg_item[CatConst.ITEMS_TAG]) == 1:
+ all_options[CatConst.ITEMS_TAG].append(cfg_item[CatConst.ITEMS_TAG][0])
+
+ return all_options
+
+
+def get_config_resp_all():
+ desired_configs = {}
+ config_all_properties_url = Options.CLUSTER_URL + "/configurations?fields=properties"
+ desired_configs_resp = curl(Options.CLUSTER_URL + "?fields=Clusters/desired_configs", validate=True, parse=True)
+
+ if 'Clusters' in desired_configs_resp:
+ if 'desired_configs' in desired_configs_resp['Clusters']:
+ desired_configs_resp = desired_configs_resp['Clusters']['desired_configs']
+ else:
+ return None
+ else:
+ return None
+
+ if Options.isPropertyAttributesSupported():
+ config_all_properties_url += ",properties_attributes"
+ all_options = curl(config_all_properties_url, validate=True, parse=True)
+ else:
+ all_options = build_all_options(desired_configs_resp)
+
+ if CatConst.ITEMS_TAG in all_options:
+ all_options = all_options[CatConst.ITEMS_TAG]
+ else:
+ return None
+
+ all_options = filter(
+ lambda x: x[CatConst.TYPE_TAG] in desired_configs_resp and x["tag"] == desired_configs_resp[x[CatConst.TYPE_TAG]][
+ "tag"],
+ all_options)
+
+ for item in all_options:
+ dc_item = {}
+
+ if CatConst.STACK_PROPERTIES in item: # config item could not contain any property
+ dc_item[CatConst.STACK_PROPERTIES] = item[CatConst.STACK_PROPERTIES]
+ else:
+ dc_item[CatConst.STACK_PROPERTIES] = {}
+
+ if CatConst.STACK_PROPERTIES_ATTRIBUTES in item:
+ dc_item[CatConst.STACK_PROPERTIES_ATTRIBUTES] = item[CatConst.STACK_PROPERTIES_ATTRIBUTES]
+
+ if "tag" in item:
+ dc_item["tag"] = item["tag"]
+
+ if dc_item != {}:
+ desired_configs[item[CatConst.TYPE_TAG]] = dc_item
+
+ return desired_configs
+
+
+def is_services_exists(required_services):
+ """
+ return true, if required_services is a part of Options.SERVICES
+ :param required_services: list
+ :return: bool
+ """
+ # sets are equal
+ if Options.SERVICES == set(required_services):
+ return True
+
+ return set(map(lambda x: x.upper(), required_services)) < Options.SERVICES
+
+
+def get_cluster_services():
+ services_url = Options.CLUSTER_URL + '/services'
+ raw_services = curl(services_url, parse=True)
+
+ # expected structure:
+ # items: [ {"href":"...", "ServiceInfo":{"cluster_name":"..", "service_name":".."}}, ..., ... ]
+ if raw_services is not None and "items" in raw_services and isinstance(raw_services["items"], list):
+ return list(map(lambda item: item["ServiceInfo"]["service_name"], raw_services["items"]))
+
+ Options.logger.warning("Failed to load services list, functionality that depends on them couldn't work")
+ return []
+
+
+def get_zookeeper_quorum():
+ zoo_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.ZOOKEEPER_SERVER), validate=False, parse=True)
+ zoo_quorum = []
+ zoo_def_port = "2181"
+ if Options.server_config_factory is not None and Options.ZK_OPTIONS in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.ZK_OPTIONS)
+ if Options.ZK_CLIENTPORT in props.properties:
+ zoo_def_port = props.properties[Options.ZK_CLIENTPORT]
+
+ if "host_components" in zoo_cfg:
+ for item in zoo_cfg["host_components"]:
+ zoo_quorum.append("%s:%s" % (item["HostRoles"]["host_name"], zoo_def_port))
+
+ return ",".join(zoo_quorum)
+
+
+def get_tez_history_url_base():
+ try:
+ tez_view = curl(Options.TEZ_VIEW_URL, validate=False, parse=True)
+ except HTTPError as e:
+ raise TemplateProcessingException(str(e))
+
+ version = ""
+ if "versions" in tez_view and \
+ len(tez_view['versions']) > 0 and \
+ "ViewVersionInfo" in tez_view['versions'][0] and \
+ 'version' in tez_view['versions'][0]['ViewVersionInfo']:
+ version = tez_view['versions'][0]['ViewVersionInfo']['version']
+ url = '{0}://{1}:{2}/#/main/views/TEZ/{3}/TEZ_CLUSTER_INSTANCE'.format(Options.API_PROTOCOL, Options.HOST, Options.API_PORT, version)
+ return url
+
+def get_kafka_listeners():
+ kafka_host = "localhost"
+ kafka_port = "6667"
+ if Options.server_config_factory is not None and Options.KAFKA_BROKER_CONF in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.KAFKA_BROKER_CONF)
+ if Options.KAFKA_PORT in props.properties:
+ kafka_port = props.properties[Options.KAFKA_PORT]
+
+ # Default kafka listeners string
+ kafka_listeners = "PLAINTEXT://{0}:{1}".format(kafka_host, kafka_port)
+
+ return kafka_listeners
+
+
+def get_ranger_xaaudit_hdfs_destination_directory():
+ namenode_hostname="localhost"
+ namenode_cfg = curl(Options.COMPONENTS_FORMAT.format(Options.NAMENODE), validate=False, parse=True)
+ if "host_components" in namenode_cfg:
+ namenode_hostname = namenode_cfg["host_components"][0]["HostRoles"]["host_name"]
+
+ return "hdfs://{0}:8020/ranger/audit".format(namenode_hostname)
+
+def get_ranger_policymgr_external_url():
+ url = "{{policymgr_mgr_url}}"
+ if Options.server_config_factory is not None and Options.RANGER_ADMIN in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.RANGER_ADMIN)
+ if Options.RANGER_EXTERNAL_URL in props.properties:
+ url = props.properties[Options.RANGER_EXTERNAL_URL]
+ return url
+
+def get_jdbc_driver():
+ driver = "{{jdbc_driver}}"
+ if Options.server_config_factory is not None and Options.RANGER_ADMIN in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.RANGER_ADMIN)
+ if "DB_FLAVOR" in props.properties:
+ db = props.properties["DB_FLAVOR"]
+
+ if db.lower() == "mysql":
+ driver = "com.mysql.jdbc.Driver"
+ elif db.lower() == "oracle":
+ driver = "oracle.jdbc.OracleDriver"
+ return driver
+
+def get_audit_jdbc_url():
+ audit_jdbc_url = "{{audit_jdbc_url}}"
+ if Options.server_config_factory is not None and Options.RANGER_ADMIN in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.RANGER_ADMIN)
+ if "DB_FLAVOR" in props.properties:
+ xa_audit_db_flavor = props.properties["DB_FLAVOR"]
+ if "db_host" in props.properties:
+ xa_db_host = props.properties["db_host"]
+ if "audit_db_name" in props.properties:
+ xa_audit_db_name = props.properties["audit_db_name"]
+
+ if xa_audit_db_flavor.lower() == 'mysql':
+ audit_jdbc_url = "jdbc:mysql://{0}/{1}".format(xa_db_host, xa_audit_db_name)
+ elif xa_audit_db_flavor.lower() == 'oracle':
+ audit_jdbc_url = "jdbc:oracle:thin:\@//{0}".format(xa_db_host)
+ return audit_jdbc_url
+
+def get_audit_db_passwd():
+ audit_db_passwd = "crypted"
+ if Options.server_config_factory is not None and Options.RANGER_ADMIN in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.RANGER_ADMIN)
+ if "audit_db_password" in props.properties:
+ audit_db_passwd = props.properties['audit_db_password']
+ return audit_db_passwd
+
+def get_audit_to_db_enabled(config_name):
+ audit_to_db = "true"
+ if Options.server_config_factory is not None and config_name in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(config_name)
+ if "XAAUDIT.DB.IS_ENABLED" in props.properties:
+ audit_to_db = props.properties["XAAUDIT.DB.IS_ENABLED"]
+ return audit_to_db
+
+def get_audit_to_hdfs_enabled(config_name):
+ audit_to_hdfs = "false"
+ if Options.server_config_factory is not None and config_name in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(config_name)
+ if "XAAUDIT.HDFS.IS_ENABLED" in props.properties:
+ audit_to_hdfs = props.properties["XAAUDIT.HDFS.IS_ENABLED"]
+ return audit_to_hdfs
+
+def get_hdfs_batch_filespool_dir(config_name, component):
+ if component == 'hdfs':
+ path = '/var/log/hadoop/hdfs/audit/hdfs/spool'
+ else:
+ path = '/var/log/{0}/audit/hdfs/spool'.format(component)
+ if Options.server_config_factory is not None and config_name in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(config_name)
+ if "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" in props.properties:
+ path = props.properties["XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY"]
+ return path
+
+
+def get_usersync_sync_source():
+ ug_sync_source = 'org.apache.ranger.unixusersync.process.UnixUserGroupBuilder'
+ sync_source = 'unix'
+ if Options.server_config_factory is not None and Options.RANGER_USERSYNC in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.RANGER_USERSYNC)
+ if "SYNC_SOURCE" in props.properties:
+ sync_source = props.properties['SYNC_SOURCE']
+
+ if sync_source == 'ldap':
+ ug_sync_source = 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder'
+ return ug_sync_source
+
+def get_audit_check(audit_type):
+ audit_check_flag = "false"
+ if Options.server_config_factory is not None and Options.RANGER_ENV in Options.server_config_factory.items():
+ props = Options.server_config_factory.get_config(Options.RANGER_ENV)
+ audit_property = "xasecure.audit.destination.{0}".format(audit_type)
+ if audit_property in props.properties:
+ audit_check_flag = props.properties[audit_property]
+
+ return audit_check_flag
+
+def get_jt_host(catalog):
+ """
+ :type catalog: UpgradeCatalog
+ :rtype str
+ """
+ if catalog.get_parsed_version()["from"] == 13:
+ return read_mapping()["JOBTRACKER"][0]
+
+ return ""
+
+
+def get_jh_host(catalog):
+ """
+ :type catalog: UpgradeCatalog
+ :rtype str
+ """
+ if catalog.get_parsed_version()["from"] == 13:
+ return read_mapping()["HISTORYSERVER"][0]
+
+ return ""
+
+def get_ranger_host():
+ ranger_config = curl(Options.COMPONENTS_FORMAT.format('RANGER_ADMIN'), validate=False, parse=True)
+ ranger_host_list = []
+ if "host_components" in ranger_config:
+ for item in ranger_config["host_components"]:
+ ranger_host_list.append(item["HostRoles"]["host_name"])
+ return ranger_host_list[0]
+
+def get_ranger_service_details():
+ server_cfg_factory = Options.server_config_factory
+ server_cfg_catalog = server_cfg_factory.get_config('admin-properties')
+ properties_latest = server_cfg_catalog.properties
+ data = {}
+
+ if properties_latest['DB_FLAVOR'].lower() == 'mysql':
+ data['RANGER_JDBC_DRIVER'] = 'com.mysql.jdbc.Driver'
+ data['RANGER_JDBC_DIALECT'] = 'org.eclipse.persistence.platform.database.MySQLPlatform'
+ data['RANGER_JDBC_URL'] = 'jdbc:mysql://{0}/{1}'.format(properties_latest['db_host'], properties_latest['db_name'])
+ data['RANGER_AUDIT_JDBC_URL'] = 'jdbc:mysql://{0}/{1}'.format(properties_latest['db_host'], properties_latest['audit_db_name'])
+ data['RANGER_ROOT_JDBC_URL'] = 'jdbc:mysql://{0}'.format(properties_latest['db_host'])
+ elif properties_latest['DB_FLAVOR'].lower() == 'oracle':
+ data['RANGER_JDBC_DRIVER'] = 'oracle.jdbc.OracleDriver'
+ data['RANGER_JDBC_DIALECT'] = 'org.eclipse.persistence.platform.database.OraclePlatform'
+ data['RANGER_JDBC_URL'] = 'jdbc:oracle:thin:@//{0}'.format(properties_latest['db_host'])
+ data['RANGER_AUDIT_JDBC_URL'] = 'jdbc:oracle:thin:@//{0}'.format(properties_latest['db_host'])
+ data['RANGER_ROOT_JDBC_URL'] = 'jdbc:oracle:thin:@//{0}'.format(properties_latest['db_host'])
+
+ return data
+
+def get_hive_security_authorization_setting():
+ # this pattern should be used only once, changes here mimic UpgradeCatalog210.java -> updateRangerHiveConfigs
+ scf = Options.server_config_factory
+ response = "None"
+
+ if "hive-env" in scf.items() and "hive_security_authorization" in scf.get_config("hive-env").properties:
+ response = scf.get_config("hive-env").properties["hive_security_authorization"]
+
+ old_ranger_catalog = "ranger-hive-plugin-properties"
+ old_ranger_setting = "ranger-hive-plugin-enabled"
+ hive_server_catalog = "hiveserver2-site"
+ hive_sec_property = "hive.security.authorization.enabled"
+
+ if scf is not None and old_ranger_catalog in scf.items():
+ cfg = scf.get_config(old_ranger_catalog)
+ prop = cfg.properties
+ if old_ranger_setting in prop and cfg.properties[old_ranger_setting].upper() == "YES":
+ response = "Ranger"
+ if hive_server_catalog in scf.items():
+ hive_props = scf.get_config(hive_server_catalog).properties
+ hive_props[hive_sec_property] = "true"
+ if old_ranger_setting in prop:
+ del prop[old_ranger_setting]
+
+ # workaround for buggy stack advisor
+ if "HIVE" in Options.SERVICES and response == "None":
+ if hive_server_catalog not in scf.items():
+ scf.create_config(hive_server_catalog)
+
+ scf.get_config(hive_server_catalog).properties[hive_sec_property] = "false"
+
+ return response
+
+
+def get_hbase_coprocessmaster_classes():
+ scf = Options.server_config_factory
+ prop = "hbase.coprocessor.master.classes"
+ hbase_ranger_enabled = False
+ old_value = ""
+ if "hbase-site" in scf.items():
+ if prop in scf.get_config("hbase-site").properties:
+ old_value = scf.get_config("hbase-site").properties[prop]
+ if "hbase.security.authorization" in scf.get_config("hbase-site").properties and \
+ scf.get_config("hbase-site").properties["hbase.security.authorization"].upper() == "TRUE":
+ hbase_ranger_enabled = True
+
+ if hbase_ranger_enabled and "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" not in old_value:
+ if "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" in old_value:
+ old_value = old_value.replace("com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor",
+ "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor")
+ else:
+ val = [] if old_value.strip() == "" else old_value.split(',')
+ val.append("org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor")
+ old_value = ','.join(val)
+
+ return old_value
+
+
+def get_rpc_scheduler_factory_class():
+ if Options.PHOENIX_QUERY_SERVER in Options.ambari_server.components:
+ return "org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory"
+ else:
+ return ""
+
+
+def get_hbase_rpc_controllerfactory_class():
+ if Options.PHOENIX_QUERY_SERVER in Options.ambari_server.components:
+ return "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory"
+ else:
+ return ""
+
+
+def get_hbase_regionserver_wal_codec():
+ prop = "phoenix_sql_enabled"
+ scf = Options.server_config_factory
+ if "hbase-env" in scf.items():
+ if prop in scf.get_config("hbase-env").properties and scf.get_config("hbase-env").properties[prop].upper() == "TRUE":
+ return "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec"
+ return "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec"
+
+
+def get_hbase_coprocessor_region_classes():
+ scf = Options.server_config_factory
+ prop = "hbase.coprocessor.region.classes"
+ hbase_ranger_enabled = False
+ old_value = ""
+ if "hbase-site" in scf.items():
+ if prop in scf.get_config("hbase-site").properties:
+ old_value = scf.get_config("hbase-site").properties[prop]
+ if "hbase.security.authorization" in scf.get_config("hbase-site").properties and \
+ scf.get_config("hbase-site").properties["hbase.security.authorization"].upper() == "TRUE":
+ hbase_ranger_enabled = True
+
+ if hbase_ranger_enabled and "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor" not in old_value:
+ if "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor" in old_value:
+ old_value = old_value.replace("com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor",
+ "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor")
+ else:
+ val = [] if old_value.strip() == "" else old_value.split(',')
+ val.append("org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor")
+ old_value = ','.join(val)
+
+ return old_value
+
+
+def _substitute_handler(upgrade_catalog, tokens, value):
+ """
+ Substitute handler
+ :param upgrade_catalog: UpgradeCatalog
+ :param tokens: list
+ :param value: str
+ :rtype str
+ """
+ for token in tokens:
+ if token == "{JOBHISTORY_HOST}":
+ value = value.replace(token, get_jh_host(upgrade_catalog))
+ elif token == "{RESOURCEMANAGER_HOST}":
+ value = value.replace(token, get_jt_host(upgrade_catalog))
+ elif token == "{HBASE_REGIONSERVER_WAL_CODEC}":
+ value = value.replace(token, get_hbase_regionserver_wal_codec())
+ elif token == "{HBASE_REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS}":
+ value = value.replace(token, get_rpc_scheduler_factory_class())
+ elif token == "{HBASE_RPC_CONTROLLERFACTORY_CLASS}":
+ value = value.replace(token, get_hbase_rpc_controllerfactory_class())
+ elif token == "{ZOOKEEPER_QUORUM}":
+ value = value.replace(token, get_zookeeper_quorum())
+ elif token == "{HBASE_COPROCESS_MASTER_CLASSES}":
+ value = value.replace(token, get_hbase_coprocessmaster_classes())
+ elif token == "{HBASE_COPROCESSOR_REGION_CLASSES}":
+ value = value.replace(token, get_hbase_coprocessor_region_classes())
+ elif token == "{HIVE_SECURITY_AUTHORIZATION}":
+ value = value.replace(token, get_hive_security_authorization_setting())
+ elif token == "{TEZ_HISTORY_URL_BASE}":
+ value = value.replace(token, get_tez_history_url_base())
+ elif token == "{RANGER_JDBC_DRIVER}":
+ value = value.replace(token, get_ranger_service_details()['RANGER_JDBC_DRIVER'])
+ elif token == "{RANGER_JDBC_URL}":
+ value = value.replace(token, get_ranger_service_details()['RANGER_JDBC_URL'])
+ elif token == "{RANGER_AUDIT_JDBC_URL}":
+ value = value.replace(token, get_ranger_service_details()['RANGER_AUDIT_JDBC_URL'])
+ elif token == "{RANGER_HOST}":
+ value = value.replace(token, get_ranger_host())
+ elif token == "{RANGER_JDBC_DIALECT}":
+ value = value.replace(token, get_ranger_service_details()['RANGER_JDBC_DIALECT'])
+ elif token == "{KAFKA_LISTENERS}":
+ value = value.replace(token, get_kafka_listeners())
+ elif token == "{RANGER_PLUGIN_HBASE_POLICY_CACHE_DIR}":
+ value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_hbase"))
+ elif token == "{RANGER_PLUGIN_HDFS_POLICY_CACHE_DIR}":
+ value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_hadoop"))
+ elif token == "{RANGER_PLUGIN_HIVE_POLICY_CACHE_DIR}":
+ value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_hive"))
+ elif token == "{RANGER_PLUGIN_KNOX_POLICY_CACHE_DIR}":
+ value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_knox"))
+ elif token == "{RANGER_PLUGIN_STORM_POLICY_CACHE_DIR}":
+ value = value.replace(token, "/etc/ranger/{0}{1}/policycache".format(Options.CLUSTER_NAME, "_storm"))
+ elif token == "{RANGER_HBASE_KEYSTORE_CREDENTIAL_FILE}":
+ value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_hbase"))
+ elif token == "{RANGER_HDFS_KEYSTORE_CREDENTIAL_FILE}":
+ value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_hadoop"))
+ elif token == "{RANGER_HIVE_KEYSTORE_CREDENTIAL_FILE}":
+ value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_hive"))
+ elif token == "{RANGER_KNOX_KEYSTORE_CREDENTIAL_FILE}":
+ value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_knox"))
+ elif token == "{RANGER_STORM_KEYSTORE_CREDENTIAL_FILE}":
+ value = value.replace(token, "jceks://file/etc/ranger/{0}{1}/cred.jceks".format(Options.CLUSTER_NAME, "_storm"))
+ elif token == "{XAAUDIT_HDFS_DESTINATION_DIRECTORY}":
+ value = value.replace(token, get_ranger_xaaudit_hdfs_destination_directory())
+ elif token == "{HBASE_RANGER_REPO_NAME}":
+ value = value.replace(token, Options.CLUSTER_NAME+"_hbase")
+ elif token == "{HDFS_RANGER_REPO_NAME}":
+ value = value.replace(token, Options.CLUSTER_NAME+"_hadoop")
+ elif token == "{HIVE_RANGER_REPO_NAME}":
+ value = value.replace(token, Options.CLUSTER_NAME+"_hive")
+ elif token == "{KNOX_RANGER_REPO_NAME}":
+ value = value.replace(token, Options.CLUSTER_NAME+"_knox")
+ elif token == "{STORM_RANGER_REPO_NAME}":
+ value = value.replace(token, Options.CLUSTER_NAME+"_storm")
+ elif token == "{POLICYMGR_MGR_URL}":
+ value = value.replace(token, get_ranger_policymgr_external_url())
+ elif token == "{HDFS_JDBC_DRIVER}":
+ value = value.replace(token, get_jdbc_driver())
+ elif token == "{HBASE_JDBC_DRIVER}":
+ value = value.replace(token, get_jdbc_driver())
+ elif token == "{HIVE_JDBC_DRIVER}":
+ value = value.replace(token, get_jdbc_driver())
+ elif token == "{KNOX_JDBC_DRIVER}":
+ value = value.replace(token, get_jdbc_driver())
+ elif token == "{STORM_JDBC_DRIVER}":
+ value = value.replace(token, get_jdbc_driver())
+ elif token == "{HDFS_AUDIT_JDBC_URL}":
+ value = value.replace(token, get_audit_jdbc_url())
+ elif token == "{HBASE_AUDIT_JDBC_URL}":
+ value = value.replace(token, get_audit_jdbc_url())
+ elif token == "{HIVE_AUDIT_JDBC_URL}":
+ value = value.replace(token, get_audit_jdbc_url())
+ elif token == "{KNOX_AUDIT_JDBC_URL}":
+ value = value.replace(token, get_audit_jdbc_url())
+ elif token == "{STORM_AUDIT_JDBC_URL}":
+ value = value.replace(token, get_audit_jdbc_url())
+ elif token == "{AUDIT_TO_DB_HDFS}":
+ value = value.replace(token, get_audit_to_db_enabled("ranger-hdfs-plugin-properties"))
+ elif token == "{AUDIT_TO_DB_HBASE}":
+ value = value.replace(token, get_audit_to_db_enabled("ranger-hbase-plugin-properties"))
+ elif token == "{AUDIT_TO_DB_HIVE}":
+ value = value.replace(token, get_audit_to_db_enabled("ranger-hive-plugin-properties"))
+ elif token == "{AUDIT_TO_DB_KNOX}":
+ value = value.replace(token, get_audit_to_db_enabled("ranger-knox-plugin-properties"))
+ elif token == "{AUDIT_TO_DB_STORM}":
+ value = value.replace(token, get_audit_to_db_enabled("ranger-storm-plugin-properties"))
+ elif token == "{AUDIT_TO_HDFS_HDFS}":
+ value = value.replace(token, get_audit_to_hdfs_enabled("ranger-hdfs-plugin-properties"))
+ elif token == "{AUDIT_TO_HDFS_HIVE}":
+ value = value.replace(token, get_audit_to_hdfs_enabled("ranger-hive-plugin-properties"))
+ elif token == "{AUDIT_TO_HDFS_HBASE}":
+ value = value.replace(token, get_audit_to_hdfs_enabled("ranger-hbase-plugin-properties"))
+ elif token == "{AUDIT_TO_HDFS_KNOX}":
+ value = value.replace(token, get_audit_to_hdfs_enabled("ranger-knox-plugin-properties"))
+ elif token == "{AUDIT_TO_HDFS_STORM}":
+ value = value.replace(token, get_audit_to_hdfs_enabled("ranger-storm-plugin-properties"))
+ elif token == "{AUDIT_HDFS_FILESPOOL_DIR_HDFS}":
+ value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-hdfs-plugin-properties", "hdfs"))
+ elif token == "{AUDIT_HDFS_FILESPOOL_DIR_HIVE}":
+ value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-hive-plugin-properties", "hive"))
+ elif token == "{AUDIT_HDFS_FILESPOOL_DIR_HBASE}":
+ value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-hbase-plugin-properties", "hbase"))
+ elif token == "{AUDIT_HDFS_FILESPOOL_DIR_KNOX}":
+ value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-knox-plugin-properties", "knox"))
+ elif token == "{AUDIT_HDFS_FILESPOOL_DIR_STORM}":
+ value = value.replace(token, get_hdfs_batch_filespool_dir("ranger-storm-plugin-properties", "storm"))
+ elif token == "{USERSYNC_SYNC_SOURCE}":
+ value = value.replace(token, get_usersync_sync_source())
+ elif token == "{AUDIT_TO_DB}":
+ value = value.replace(token, get_audit_check("db"))
+ elif token == "{AUDIT_TO_HDFS}":
+ value = value.replace(token, get_audit_check("hdfs"))
+ elif token == "{RANGER_ROOT_JDBC_URL}":
+ value = value.replace(token, get_ranger_service_details()['RANGER_ROOT_JDBC_URL'])
+
+ return value
+
+
+def modify_config_item(config_type, catalog, server_config_factory):
+ """
+ Modify configuration item
+ :type config_type str
+ :type catalog UpgradeCatalog
+ :type server_config_factory ServerConfigFactory
+ """
+
+ # if config group is absent on the server, we will create it
+ if config_type not in server_config_factory.items():
+ server_config_factory.create_config(config_type)
+
+ server_config_catalog = server_config_factory.get_config(config_type)
+
+ server_config_catalog.merge(catalog)
+
+
+def modify_configs():
+ if len(Options.ARGS) > 1:
+ config_type = Options.ARGS[1]
+ else:
+ config_type = None
+
+ catalog_farm = UpgradeCatalogFactory(Options.OPTIONS.upgrade_json) # Load upgrade catalog
+ catalog = catalog_farm.get_catalog(Options.OPTIONS.from_stack,
+ Options.OPTIONS.to_stack) # get desired version of catalog
+ Options.stack_advisor = StackAdvisorFactory().get_instance(catalog.name, catalog.target_version)
+
+ # load all desired configs from the server
+ # ToDo: implement singleton for that class
+ Options.server_config_factory = ServerConfigFactory()
+
+ if catalog is None:
+ raise FatalException(1, "Upgrade catalog for version %s-%s not found, no configs was modified"
+ % (Options.OPTIONS.from_stack, Options.OPTIONS.to_stack))
+
+ # add user-defined template processing function
+ catalog.add_handler(CatConst.TEMPLATE_HANDLER, _substitute_handler)
+
+ if config_type is not None and config_type not in catalog.items:
+ raise FatalException("Config type %s not exists, no configs was modified" % config_type)
+
+ if config_type is not None:
+ modify_config_item(config_type, catalog, Options.server_config_factory)
+ else:
+ for collection_name in catalog.items:
+ modify_config_item(collection_name, catalog, Options.server_config_factory)
+
+ Options.server_config_factory.process_mapping_transformations(catalog)
+
+ # commit changes to server, if any will be found
+ Options.server_config_factory.commit()
+
+
+def backup_configs(conf_type=None):
+ dir = "backups_%d" % time.time()
+ file_pattern = "%s%s%s_%s.json"
+ configs = get_config_resp_all()
+ if configs is None:
+ Options.logger.error("Unexpected response from the server")
+ return -1
+
+ if conf_type is not None and conf_type in configs:
+ configs = {conf_type: configs[conf_type]}
+
+ if not os.path.exists(dir):
+ os.mkdir(dir)
+
+ for item in configs:
+ filename = file_pattern % (dir, os.path.sep, item, configs[item]["tag"])
+ if os.path.exists(filename):
+ os.remove(filename)
+
+ try:
+ with open(filename, "w") as f:
+ f.write(json.dumps(configs[item][CatConst.STACK_PROPERTIES], indent=4))
+ Options.logger.info("Catalog \"%s\" stored to %s", item, filename)
+ except IOError as e:
+ Options.logger.error("Unable to store \"%s\": %s", item, e)
+
+
+def install_services():
+ SERVICE_URL_FORMAT = Options.CLUSTER_URL + '/services/{0}'
+ SERVICES = ["MAPREDUCE2", "YARN"]
+ PUT_IN_INSTALLED = [
+ {
+ "RequestInfo": {
+ "context": "Install MapReduce2"
+ },
+ "Body": {
+ "ServiceInfo": {
+ "state": "INSTALLED"
+ }
+ }
+ },
+ {
+ "RequestInfo": {
+ "context": "Install YARN"
+ },
+ "Body": {
+ "ServiceInfo": {
+ "state": "INSTALLED"
+ }
+ }
+ }
+ ]
+
+ err_retcode = 0
+ err_message = ""
+ for index in [0, 1]:
+ try:
+ curl(SERVICE_URL_FORMAT.format(SERVICES[index]), validate=True, request_type="PUT", data=PUT_IN_INSTALLED[index])
+ except FatalException as e:
+ if not e.code == 0:
+ err_retcode = e.code
+ err_message = err_message + " Error while installing " + SERVICES[index] + ". Details: " + e.message + "."
+
+ if err_retcode != 0:
+ raise FatalException(err_retcode,
+ err_message + "(Services may already be installed or agents are not yet started.)")
+
+ Options.OPTIONS.exit_message = "Requests has been submitted to install YARN and MAPREDUCE2. Use Ambari Web to monitor " \
+ "the status of the install requests."
+
+
+def generate_auth_header(user, password):
+ token = "%s:%s" % (user, password)
+ token = base64.encodestring(token)
+ return {"Authorization": "Basic %s" % token.replace('\n', '')}
+
+
+def curl(url, tokens=None, headers=None, request_type="GET", data=None, parse=False,
+ validate=False, soft_validation=False):
+ """
+ :rtype type
+ """
+ _headers = {}
+ handler_chain = []
+ post_req = ["POST", "PUT"]
+ get_req = ["GET", "DELETE"]
+
+ print_url = Options.CURL_PRINT_ONLY is not None
+ write_only_print = Options.CURL_WRITE_ONLY is not None
+
+ if request_type not in post_req + get_req:
+ raise IOError("Wrong request type \"%s\" passed" % request_type)
+
+ if data is not None and isinstance(data, dict):
+ data = json.dumps(data)
+
+ if tokens is not None:
+ _headers.update(generate_auth_header(tokens["user"], tokens["pass"]))
+ elif Options.API_TOKENS is not None:
+ _headers.update(generate_auth_header(Options.API_TOKENS["user"], Options.API_TOKENS["pass"]))
+
+ if request_type in post_req and data is not None:
+ _headers["Content-Length"] = len(data)
+
+ if headers is not None:
+ _headers.update(headers)
+
+ if Options.HEADERS is not None:
+ _headers.update(Options.HEADERS)
+
+ director = build_opener(*handler_chain)
+ if request_type in post_req:
+ _data = bytes(data)
+ req = Request(url, headers=_headers, data=_data)
+ else:
+ req = Request(url, headers=_headers)
+
+ req.get_method = lambda: request_type
+
+ if print_url:
+ if write_only_print:
+ if request_type in post_req:
+ Options.logger.info(url)
+ if data is not None:
+ Options.logger.info("POST Data: \n" + str(data))
+ else:
+ Options.logger.info(url)
+ if request_type in post_req and data is not None:
+ Options.logger.info("POST Data: \n" + str(data))
+
+ code = 200
+ if not (print_url and request_type in post_req):
+ try:
+ resp = director.open(req)
+ out = resp.read()
+ if isinstance(out, bytes):
+ out = out.decode("utf-8")
+ code = resp.code
+ except URLError as e:
+ Options.logger.error(str(e))
+ if isinstance(e, HTTPError):
+ raise e
+ else:
+ raise FatalException(-1, str(e))
+ else:
+ if not print_url:
+ Options.logger.info(url)
+ out = "{}"
+
+ if validate and not print_url and (code > 299 or code < 200):
+ if soft_validation:
+ Options.logger.warning("Response validation failed, please check previous action result manually.")
+ else:
+ raise FatalException(code, "Response validation failed, please check previous action result manually.")
+
+ if parse:
+ return json.loads(out)
+ else:
+ return out
+
+
+def configuration_item_diff(collection_name, catalog, actual_properties_list):
+ """
+ Merge catalog item with actual config item on the server
+ Diff item response:
+ {
+ "property" : name,
+ "catalog_item": value,
+ "catalog_value": value,
+ "actual_value": value
+ }
+ :param collection_name:
+ :param catalog:
+ :param actual_properties_list
+ :return:
+ """
+
+ verified_catalog = []
+ catalog_properties = dict(catalog)
+ actual_properties = dict(actual_properties_list)
+
+ if actual_properties is None:
+ verified_catalog = map(lambda x: {
+ "property": x,
+ "catalog_item": catalog_properties[x],
+ "catalog_value": catalog_properties[x][CatConst.PROPERTY_VALUE_TAG],
+ "actual_value": None
+ }, catalog_properties.keys())
+ else:
+ # build list of properties according to actual properties
+ verified_catalog = map(lambda x: {
+ "property": x,
+ "catalog_item": catalog_properties[x] if x in catalog_properties else None,
+ "catalog_value": catalog_properties[x][CatConst.PROPERTY_VALUE_TAG] if x in catalog_properties else None,
+ "actual_value": actual_properties[x]
+ }, actual_properties.keys())
+
+ # build list of properties according to catalog properties
+ verified_catalog_catalog = map(lambda x: {
+ "property": x,
+ "catalog_item": catalog_properties[x],
+ "catalog_value": catalog_properties[x][CatConst.PROPERTY_VALUE_TAG] if CatConst.PROPERTY_VALUE_TAG in
+ catalog_properties[x] else None,
+ "actual_value": actual_properties[x] if x in actual_properties else None,
+ }, catalog_properties.keys())
+
+ # append properties, which are listened in catalog but doesn't present in the actual configuration
+ verified_catalog += filter(lambda x: x["property"] not in actual_properties, verified_catalog_catalog)
+
+ return verified_catalog
+
+
+def configuration_diff_analyze(diff_list):
+ report = {}
+ for item_key in diff_list.keys():
+ property_diff_list = diff_list[item_key]
+ item_stat = {
+ "skipped": {"count": 0, "items": []},
+ "ok": {"count": 0, "items": []},
+ "fail": {"count": 0, "items": []},
+ "total": {"count": len(property_diff_list), "items": []}
+ }
+
+ def push_status(status, _property_item):
+ item_stat[status]["count"] += 1
+ item_stat[status]["items"].append(_property_item)
+
+ for property_item in property_diff_list:
+ # process properties which can be absent
+
+ # item was removed, from actual configs according to catalog instructions
+ if property_item["actual_value"] is None \
+ and CatConst.PROPERTY_REMOVE_TAG in property_item["catalog_item"] \
+ and property_item["catalog_item"][CatConst.PROPERTY_REMOVE_TAG] == CatConst.TRUE_TAG:
+
+ push_status("ok", property_item)
+
+ # currently skip values with template tag, as there no filter implemented
+ # ToDo: implement possibility to filter values without filter handler,
+ # ToDo: currently filtering is possible only on update-configs stage
+ elif property_item["actual_value"] is not None and property_item["catalog_value"] is not None \
+ and CatConst.VALUE_TEMPLATE_TAG in property_item["catalog_item"] \
+ and property_item["catalog_item"][CatConst.VALUE_TEMPLATE_TAG] == CatConst.TRUE_TAG:
+
+ push_status("skipped", property_item)
+
+ # item not present in actual config, but present in catalog and no remove tag is present
+ elif property_item["actual_value"] is None and property_item["catalog_value"] is not None:
+ push_status("fail", property_item)
+
+ # property exists in actual configuration, but not described in catalog configuration
+ elif property_item["actual_value"] is not None and property_item["catalog_value"] is None:
+ push_status("skipped", property_item)
+
+ # actual and catalog properties are equal
+ elif property_item["catalog_value"] == property_item["actual_value"]:
+ push_status("ok", property_item)
+ elif property_item["catalog_value"] != property_item["actual_value"]:
+ push_status("fail", property_item)
+
+ report[item_key] = item_stat
+ return report
+
+
+def verify_configuration():
+ diff_list = {}
+
+ if len(Options.ARGS) > 1:
+ config_type = Options.ARGS[1]
+ else:
+ config_type = None
+
+ catalog_farm = UpgradeCatalogFactory(Options.OPTIONS.upgrade_json) # Load upgrade catalog
+ catalog = catalog_farm.get_catalog(Options.OPTIONS.from_stack,
+ Options.OPTIONS.to_stack) # get desired version of catalog
+ server_configs = ServerConfigFactory()
+
+ if catalog is None:
+ raise FatalException(1, "Upgrade catalog for version %s-%s not found"
+ % (Options.OPTIONS.from_stack, Options.OPTIONS.to_stack))
+
+ if config_type is not None and config_type not in catalog.items.keys() and config_type not in server_configs.items():
+ raise FatalException("Config type %s not exists" % config_type)
+
+ # fetch from server all option at one time and filter only desired versions
+
+ if config_type is not None:
+ diff_list[config_type] = configuration_item_diff(config_type, catalog.items[config_type], server_configs.get_config(config_type).properties)
+ else:
+ for collection_name in catalog.items.keys():
+ diff_list[collection_name] = configuration_item_diff(collection_name, catalog.items[collection_name], server_configs.get_config(collection_name).properties)
+
+ analyzed_list = configuration_diff_analyze(diff_list)
+
+ report_file = None
+ if Options.REPORT_FILE is not None:
+ try:
+ report_file = open(Options.REPORT_FILE, "w")
+ except IOError as e:
+ Options.logger.error("Report file open error: %s" % e.message)
+
+ for config_item in analyzed_list:
+ if analyzed_list[config_item]["fail"]["count"] != 0:
+ Options.logger.info(
+ "%s: %s missing configuration(s) - please look in the output file for the missing params" % (
+ config_item, analyzed_list[config_item]["fail"]["count"]
+ )
+ )
+ if report_file is not None:
+ report_formatter(report_file, config_item, analyzed_list[config_item])
+ else:
+ Options.logger.info("%s: verified" % config_item)
+
+ if report_file is not None:
+ try:
+ report_file.close()
+ except IOError as e:
+ Options.logger.error("Report file close error: %s" % e.message)
+
+
+def report_formatter(report_file, config_item, analyzed_list_item):
+ prefix = "Configuration item %s" % config_item
+ if analyzed_list_item["fail"]["count"] > 0:
+ for item in analyzed_list_item["fail"]["items"]:
+ report_file.write("%s: property \"%s\" is set to \"%s\", but should be set to \"%s\"\n" % (
+ prefix, item["property"], item["actual_value"], item["catalog_value"]
+ ))
+
+
+def main():
+ action_list = { # list of supported actions
+ Options.GET_MR_MAPPING_ACTION: get_mr1_mapping,
+ Options.DELETE_MR_ACTION: delete_mr,
+ Options.ADD_YARN_MR2_ACTION: add_services,
+ Options.MODIFY_CONFIG_ACTION: modify_configs,
+ Options.INSTALL_YARN_MR2_ACTION: install_services,
+ Options.BACKUP_CONFIG_ACTION: backup_configs,
+ Options.VERIFY_ACTION: verify_configuration
+ }
+
+ parser = optparse.OptionParser(usage="usage: %prog [options] action\n Valid actions: "
+ + ", ".join(action_list.keys())
+ + "\n update-configs accepts type, e.g. hdfs-site to update specific configs")
+
+ parser.add_option("-n", "--printonly",
+ action="store_true", dest="printonly", default=False,
+ help="Prints all the curl commands to be executed (no post/update request will be performed)")
+ parser.add_option("-w", "--writeonly",
+ action="store_true", dest="writeonly", default=False,
+ help="in the combination with --printonly param will print only post/update requests")
+ parser.add_option("-o", "--log", dest="logfile", default=None,
+ help="Log file")
+ parser.add_option("--report", dest="report", default=None,
+ help="Report file output location")
+
+ parser.add_option('--upgradeCatalog', default=None, help="Upgrade Catalog file full path", dest="upgrade_json")
+ parser.add_option('--fromStack', default=None, help="stack version to upgrade from", dest="from_stack")
+ parser.add_option('--toStack', default=None, help="stack version to upgrade to", dest="to_stack")
+
+ parser.add_option('--hostname', default=None, help="Hostname for Ambari server", dest="hostname")
+ parser.add_option('--port', default='8080', help="Port number for Ambari server", dest="port")
+ parser.add_option('--https', default=False, action="store_true", dest="https", help="Use https protocol for connection to the server")
+ parser.add_option('--user', default=None, help="Ambari admin user", dest="user")
+ parser.add_option('--password', default=None, help="Ambari admin password", dest="password")
+ parser.add_option('--clustername', default=None, help="Cluster name", dest="clustername")
+
+ (options, args) = parser.parse_args()
+ Options.initialize_logger(options.logfile)
+ options.warnings = []
+
+ if len(args) =
<TRUNCATED>
[13/55] [abbrv] ambari git commit: AMBARI-18184 - Hive Metastore
restart failed during EU with 'Internal credentials cache error' while
running kinit (jonathanhurley)
Posted by ol...@apache.org.
AMBARI-18184 - Hive Metastore restart failed during EU with 'Internal credentials cache error' while running kinit (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/78886384
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/78886384
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/78886384
Branch: refs/heads/branch-dev-logsearch
Commit: 78886384b075f5e0108b587dbf1760d29848641d
Parents: d3bac1d
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Aug 17 10:56:40 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Aug 18 13:01:44 2016 -0400
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/package/scripts/hive_metastore.py | 11 ++++++++---
.../HIVE/0.12.0.2.0/package/scripts/params_linux.py | 3 ++-
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/78886384/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 772eff8..17bf581 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -181,6 +181,7 @@ class HiveMetastoreDefault(HiveMetastore):
Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
"""
Logger.info("Upgrading Hive Metastore Schema")
+ import status_params
import params
env.set_params(params)
@@ -189,9 +190,13 @@ class HiveMetastoreDefault(HiveMetastore):
self.configure(env)
if params.security_enabled:
- kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
- Execute(kinit_command,user=params.smokeuser)
-
+ cached_kinit_executor(status_params.kinit_path_local,
+ status_params.hive_user,
+ params.hive_metastore_keytab_path,
+ params.hive_metastore_principal,
+ status_params.hostname,
+ status_params.tmp_dir)
+
# ensure that the JDBC drive is present for the schema tool; if it's not
# present, then download it first
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list:
http://git-wip-us.apache.org/repos/asf/ambari/blob/78886384/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 37e2426..1cd58c3 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -341,7 +341,8 @@ fs_root = config['configurations']['core-site']['fs.defaultFS']
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hive_metastore_keytab_path = config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file']
+hive_metastore_keytab_path = config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file']
+hive_metastore_principal = config['configurations']['hive-site']['hive.metastore.kerberos.principal']
hive_server2_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
[47/55] [abbrv] ambari git commit: AMBARI-18095. Use Zookeeper *.log
file instead of *.out (Dharmesh Makwana via oleewere)
Posted by ol...@apache.org.
AMBARI-18095. Use Zookeeper *.log file instead of *.out (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/34b9e406
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/34b9e406
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/34b9e406
Branch: refs/heads/branch-dev-logsearch
Commit: 34b9e4060dc3dcdeaad6812788fc5a2e020015a4
Parents: 2c7164d
Author: oleewere <ol...@gmail.com>
Authored: Thu Aug 18 13:20:59 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Aug 22 12:56:03 2016 +0200
----------------------------------------------------------------------
.../0.5.0/package/templates/input.config-zookeeper.json.j2 | 4 ++--
.../ZOOKEEPER/3.4.5/configuration/zookeeper-log4j.xml | 4 ++--
.../ZOOKEEPER/3.4.5/package/scripts/params_windows.py | 2 +-
.../common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper.py | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/34b9e406/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
index 083bd8a..5525a64 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2
@@ -20,7 +20,7 @@
{
"type":"zookeeper",
"rowtype":"service",
- "path":"{{zk_log_dir}}/zookeeper*.out"
+ "path":"{{zk_log_dir}}/zookeeper*.log"
}
],
@@ -53,4 +53,4 @@
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/34b9e406/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-log4j.xml b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-log4j.xml
index 70e86d7..04f872c 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-log4j.xml
@@ -52,7 +52,7 @@
#
# DEFAULT: console appender only
-log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=INFO, CONSOLE, ROLLINGFILE
# Example with rolling log file
#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
@@ -73,7 +73,7 @@ log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L]
# Log DEBUG level and above messages to a log file
log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
log4j.appender.ROLLINGFILE.Threshold=DEBUG
-log4j.appender.ROLLINGFILE.File=zookeeper.log
+log4j.appender.ROLLINGFILE.File={{zk_log_dir}}/zookeeper.log
# Max log file size of 10MB
log4j.appender.ROLLINGFILE.MaxFileSize=10MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/34b9e406/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/params_windows.py
index a83214d..de8cb51 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/params_windows.py
@@ -38,7 +38,7 @@ except:
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
zk_user = hadoop_user
-# notused zk_log_dir = config['configurations']['zookeeper-env']['zk_log_dir']
+zk_log_dir = config['configurations']['zookeeper-env']['zk_log_dir']
zk_data_dir = ensure_double_backslashes(config['configurations']['zoo.cfg']['dataDir'])
tickTime = config['configurations']['zoo.cfg']['tickTime']
initLimit = config['configurations']['zoo.cfg']['initLimit']
http://git-wip-us.apache.org/repos/asf/ambari/blob/34b9e406/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper.py
index 18e423c..a04d393 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper.py
@@ -95,7 +95,7 @@ def zookeeper(type = None, upgrade_type=None):
mode=0644,
group=params.user_group,
owner=params.zk_user,
- content=params.log4j_props
+ content=InlineTemplate(params.log4j_props)
)
elif (os.path.exists(os.path.join(params.config_dir, "log4j.properties"))):
File(os.path.join(params.config_dir, "log4j.properties"),
[16/55] [abbrv] ambari git commit: AMBARI-18167: RU: Kafka brokers
restart was stopped during downgrade cluster (Mugdha Varadkar via
jonathanhurley)
Posted by ol...@apache.org.
AMBARI-18167: RU: Kafka brokers restart was stopped during downgrade cluster (Mugdha Varadkar via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fda22e2e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fda22e2e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fda22e2e
Branch: refs/heads/branch-dev-logsearch
Commit: fda22e2e626e655674d55a8f1e91c70bbdb7a8ae
Parents: 712a12b
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Aug 18 14:11:44 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Aug 18 14:11:50 2016 -0400
----------------------------------------------------------------------
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 17 -----------------
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 14 +-------------
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 1 +
3 files changed, 2 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fda22e2e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 9690efc..0611301 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -479,21 +479,6 @@
</component>
</service>
- <service name="KAFKA">
- <component name="KAFKA_BROKER">
- <post-upgrade>
- <task xsi:type="configure" id ="hdp_2_4_0_0_kafka_broker_deprecate_port"/>
- </post-upgrade>
-
- <!-- no-op to prevent config changes on downgrade -->
- <pre-downgrade/>
-
- <upgrade>
- <task xsi:type="restart-task" />
- </upgrade>
- </component>
- </service>
-
<service name="RANGER">
<component name="RANGER_ADMIN">
<pre-upgrade>
@@ -875,8 +860,6 @@
<service name="KAFKA">
<component name="KAFKA_BROKER">
- <pre-downgrade/>
-
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fda22e2e/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index bc8e9f9..839e544 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -534,19 +534,6 @@
</component>
</service>
- <service name="KAFKA">
- <component name="KAFKA_BROKER">
- <pre-upgrade>
- <task xsi:type="configure" id ="hdp_2_4_0_0_kafka_broker_deprecate_port"/>
- <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
- </pre-upgrade>
-
- <upgrade>
- <task xsi:type="restart-task" />
- </upgrade>
- </component>
- </service>
-
<service name="RANGER">
<component name="RANGER_ADMIN">
<pre-downgrade/> <!-- no-op to prevent config changes on downgrade -->
@@ -969,6 +956,7 @@
<service name="KAFKA">
<component name="KAFKA_BROKER">
+ <pre-downgrade/>
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
</pre-upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fda22e2e/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index e886301..f5d7b18 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -531,6 +531,7 @@
<service name="KAFKA">
<component name="KAFKA_BROKER">
+ <pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
</pre-upgrade>
[03/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
new file mode 100644
index 0000000..70f1ffc
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
@@ -0,0 +1,440 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.1",
+ "target-version": "2.3",
+ "options": {
+ "config-types": {
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ },
+ "capacity-scheduler": {
+ "merged-copy": "yes",
+ "required-services": ["YARN"]
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "hive-env": {
+ "merged-copy": "yes",
+ "required-services": ["HIVE"]
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "tez-site":{
+ "merged-copy": "yes"
+ },
+ "falcon-startup.properties": {
+ "merged-copy": "yes"
+ },
+ "storm-site":{
+ "merged-copy": "yes"
+ },
+ "storm-env": {
+ "merged-copy": "yes",
+ "required-services": [
+ "STORM"
+ ]
+ },
+ "storm-cluster-log4j": {
+ "merged-copy": "yes",
+ "required-services": [
+ "STORM"
+ ]
+ },
+ "storm-worker-log4j": {
+ "merged-copy": "yes",
+ "required-services": [
+ "STORM"
+ ]
+ }
+ }
+ },
+ "properties": {
+ "storm-env": {
+ "nimbus_seeds_supported": "true",
+ "storm_logs_supported": "true"
+ },
+ "storm-cluster-log4j": {
+ "content": "\n \n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n\n<configuration monitorInterval=\"60\">\n<properties>\n <property name=\"pattern\">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property>\n <property name=\"patt
ernMetris\">%d %-8r %m%n</property>\n</properties>\n<appenders>\n <RollingFile name=\"A1\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.%i\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile name=\"ACCESS\"\n fileName=\"${sys:storm.log.dir}/access.log\"\n filePattern=\"${sys:storm.log.dir}/access.log.%i\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile
name=\"METRICS\"\n fileName=\"${sys:storm.log.dir}/metrics.log\"\n filePattern=\"${sys:storm.log.dir}/metrics.log.%i\">\n <PatternLayout>\n <pattern>${patternMetris}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"2 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <Syslog name=\"syslog\" format=\"RFC5424\" host=\"localhost\" port=\"514\"\n protocol=\"UDP\" appName=\"[${sys:daemon.name}]\" mdcId=\"mdc\" includeMDC=\"true\"\n facility=\"LOCAL5\" enterpriseNumber=\"18060\" newLine=\"true\" exceptionPattern=\"%rEx{full}\"\n messageId=\"[${sys:user.name}:S0]\" id=\"storm\"/>\n</appenders>\n<loggers>\n\n <Logger name=\"backtype.storm.security.auth.authorizer\" level=\"info\">\n <AppenderRef ref=\"ACCESS\"/>\n </Logger>\n <Logger name=\"backtype.storm.metric.Loggi
ngMetricsConsumer\" level=\"info\">\n <AppenderRef ref=\"METRICS\"/>\n </Logger>\n <root level=\"info\"> <!-- We log everything -->\n <appender-ref ref=\"A1\"/>\n <appender-ref ref=\"syslog\"/>\n </root>\n</loggers>\n</configuration>\n \n "
+ },
+ "storm-worker-log4j": {
+ "content": "\n \n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n\n<configuration monitorInterval=\"60\">\n<properties>\n <property name=\"pattern\">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property>\n <property name=\"patt
ernNoTime\">%msg%n</property>\n</properties>\n<appenders>\n <RollingFile name=\"A1\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.%i.gz\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile name=\"STDOUT\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}.out\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.out.%i.gz\">\n <PatternLayout>\n <pattern>${patternNoTime}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"4\"/>\n
</RollingFile>\n <RollingFile name=\"STDERR\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}.err\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.err.%i.gz\">\n <PatternLayout>\n <pattern>${patternNoTime}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"4\"/>\n </RollingFile>\n <Syslog name=\"syslog\" format=\"RFC5424\" host=\"localhost\" port=\"514\"\n protocol=\"UDP\" appName=\"[${sys:storm.id}:${sys:worker.port}]\" mdcId=\"mdc\" includeMDC=\"true\"\n facility=\"LOCAL5\" enterpriseNumber=\"18060\" newLine=\"true\" exceptionPattern=\"%rEx{full}\"\n messageId=\"[${sys:user.name}:${sys:logging.sensitivity}]\" id=\"storm\"/>\n</appenders>\n<loggers>\n <root level=\"info\"> <!-- We log everything -->\n <appender-ref ref=\"A1\"/>\n
<appender-ref ref=\"syslog\"/>\n </root>\n <Logger name=\"STDERR\" level=\"INFO\">\n <appender-ref ref=\"STDERR\"/>\n <appender-ref ref=\"syslog\"/>\n </Logger>\n <Logger name=\"STDOUT\" level=\"INFO\">\n <appender-ref ref=\"STDOUT\"/>\n <appender-ref ref=\"syslog\"/>\n </Logger>\n</loggers>\n</configuration>\n \n "
+ },
+ "storm-site": {
+ "supervisor.childopts": "-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port={{jmxremote_port}} -javaagent:/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "nimbus.childopts": "-Xmx1024m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "worker.childopts": "-Xmx768m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-client/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-client/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM"
+ },
+ "hive-env": {
+ "hive_security_authorization": {
+ "value": "{HIVE_SECURITY_AUTHORIZATION}",
+ "template": "yes",
+ "resolve-dependency": "yes"
+ }
+ },
+ "falcon-startup.properties": {
+ "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService",
+ "*.dfs.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+ "*.falcon.enableTLS": "false",
+ "*.falcon.http.authentication.cookie.domain": "EXAMPLE.COM",
+ "*.falcon.http.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "*.falcon.http.authentication.kerberos.principal": "HTTP/_HOST@EXAMPLE.COM",
+ "*.falcon.security.authorization.admin.groups": "falcon",
+ "*.falcon.security.authorization.admin.users": "falcon,ambari-qa",
+ "*.falcon.security.authorization.enabled": "false",
+ "*.falcon.security.authorization.provider": "org.apache.falcon.security.DefaultAuthorizationProvider",
+ "*.falcon.security.authorization.superusergroup": "falcon",
+ "*.falcon.service.authentication.kerberos.keytab": "/etc/security/keytabs/falcon.service.keytab",
+ "*.falcon.service.authentication.kerberos.principal": "falcon/_HOST@EXAMPLE.COM",
+ "*.journal.impl": "org.apache.falcon.transaction.SharedFileSystemJournal",
+ "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
+ "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph, org.apache.falcon.entity.ColoClusterRelation, org.apache.falcon.group.FeedGroupMap",
+ "*.shared.libs": "activemq-core,ant,geronimo-j2ee-management,jms,json-simple,oozie-client,spring-jms,commons-lang3,commons-el",
+ "*.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap,\\\n org.apache.falcon.service.SharedLibraryHostingService"
+ },
+ "tez-site":{
+ "tez.am.container.idle.release-timeout-max.millis": "20000",
+ "tez.am.container.idle.release-timeout-min.millis": "10000",
+ "tez.am.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.am.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.am.max.app.attempts": "2",
+ "tez.am.maxtaskfailures.per.node": "10",
+ "tez.cluster.additional.classpath.prefix": "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "tez.counters.max": "2000",
+ "tez.counters.max.groups": "1000",
+ "tez.generate.debug.artifacts": "false",
+ "tez.grouping.max-size": "1073741824",
+ "tez.grouping.min-size": "16777216",
+ "tez.grouping.split-waves": "1.7",
+ "tez.history.logging.service.class": "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService",
+ "tez.runtime.compress": "true",
+ "tez.runtime.compress.codec": "org.apache.hadoop.io.compress.SnappyCodec",
+ "tez.runtime.io.sort.mb": "272",
+ "tez.runtime.unordered.output.buffer.size-mb": "51",
+ "tez.shuffle-vertex-manager.max-src-fraction": "0.4",
+ "tez.shuffle-vertex-manager.min-src-fraction": "0.2",
+ "tez.task.am.heartbeat.counter.interval-ms.max": "4000",
+ "tez.task.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.task.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.task.max-events-per-heartbeat": "500",
+ "tez.task.resource.memory.mb": "682",
+ "tez.am.container.reuse.non-local-fallback.enabled": "false",
+ "tez.am.resource.memory.mb": "1364",
+ "tez.lib.uris": "/hdp/apps/${hdp.version}/tez/tez.tar.gz",
+ "tez.session.client.timeout.secs": "-1",
+ "tez.am.view-acls": "*",
+ "tez.tez-ui.history-url.base": {
+ "value": "{TEZ_HISTORY_URL_BASE}",
+ "template": "yes"
+ },
+ "tez.am.container.session.delay-allocation-millis": {"remove": "yes"},
+ "tez.am.env": {"remove": "yes"},
+ "tez.am.grouping.max-size": {"remove": "yes"},
+ "tez.am.grouping.min-size": {"remove": "yes"},
+ "tez.am.grouping.split-waves": {"remove": "yes"},
+ "tez.am.java.opt": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.max-src-fraction": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.min-src-fraction": {"remove": "yes"},
+ "tez.runtime.intermediate-input.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-input.is-compressed": {"remove": "yes"},
+ "tez.runtime.intermediate-output.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-output.should-compress": {"remove": "yes"},
+ "tez.yarn.ats.enabled": {"remove": "yes"}
+ },
+ "webhcat-site": {
+ "templeton.storage.class": "org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage",
+ "templeton.port": "50111"
+ },
+ "oozie-site": {
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.service.HadoopAccessorService.kerberos.enabled": "false",
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.AuthorizationService.authorization.enabled": "true",
+ "oozie.authentication.kerberos.name.rules": "RULE:[2:$1@$0]([jt]t@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-MAPREDUSER/\nRULE:[2:$1@$0]([nd]n@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HDFSUSER/\nRULE:[2:$1@$0](hm@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\nRULE:[2:$1@$0](rs@.*TODO-KERBEROS-DOMAIN)s/.*/TODO-HBASE-USER/\nDEFAULT",
+ "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials",
+ "oozie.service.CallableQueueService.callable.concurrency": {"remove": "yes"},
+ "oozie.service.CallableQueueService.queue.size": {"remove": "yes"},
+ "oozie.service.CallableQueueService.threads": {"remove": "yes"},
+ "oozie.service.JPAService.create.db.schema": {"remove": "yes"},
+ "oozie.service.JPAService.pool.max.active.conn": {"remove": "yes"},
+ "oozie.service.PurgeService.older.than": {"remove": "yes"},
+ "oozie.service.PurgeService.purge.interval": {"remove": "yes"},
+ "oozie.service.SchemaService.wf.ext.schemas": {"remove": "yes"},
+ "oozie.service.WorkflowAppService.system.libpath": {"remove": "yes"},
+ "oozie.service.coord.normal.default.timeout": {"remove": "yes"},
+ "oozie.service.coord.push.check.requeue.interval": {"remove": "yes"},
+ "oozie.services": {"remove": "yes"},
+ "oozie.system.id": {"remove": "yes"},
+ "oozie.systemmode": {"remove": "yes"},
+ "use.system.libpath.for.mapreduce.and.pig.jobs": {"remove": "yes"}
+
+ },
+ "hive-site": {
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.transport.mode": "binary",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.optimize.reducededuplication.min.reducer": "4",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.support.dynamic.service.discovery": "true",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "fs.file.impl.disable.cache": "true",
+ "fs.hdfs.impl.disable.cache": "true"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {"remove": "yes"},
+ "hbase.region.server.rpc.scheduler.factory.class": {
+ "value": "{HBASE_REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS}",
+ "required-services": [
+ "HBASE"
+ ],
+ "template": "yes"
+ },
+ "hbase.rpc.controllerfactory.class": {
+ "value": "{HBASE_RPC_CONTROLLERFACTORY_CLASS}",
+ "template": "yes",
+ "required-services": [
+ "HBASE"
+ ]
+ },
+ "hbase.regionserver.wal.codec": {
+ "value": "{HBASE_REGIONSERVER_WAL_CODEC}",
+ "template": "yes",
+ "required-services": [
+ "HBASE"
+ ]
+ },
+ "phoenix.functions.allowUserDefinedFunctions": "true",
+ "fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
+ "hbase.bucketcache.percentage.in.combinedcache": {"remove": "yes"},
+ "hbase.coprocessor.enabled": {
+ "value": "true",
+ "override": "yes"
+ },
+ "hbase.bulkload.staging.dir": {
+ "value": "/apps/hbase/staging",
+ "override": "no"
+ }
+ },
+ "mapred-site": {
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "mapreduce.fileoutputcommitter.algorithm.version": "1",
+ "mapreduce.task.tmp.dir": {"remove": "yes"}
+ },
+ "core-site": {
+ "hadoop.http.authentication.simple.anonymous.allowed": "true"
+ },
+ "hdfs-site": {
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600",
+ "dfs.datanode.max.transfer.threads": "4096",
+ "dfs.namenode.inode.attributes.provider.class": {
+ "value": "org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer",
+ "required-services": ["RANGER"]
+ }
+ },
+ "yarn-site": {
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.client.nodemanager-connect.max-wait-ms": "900000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strictresource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-er-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "/var/log/hadoop-yarn/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "false",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "false",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-address": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ },
+ "hadoop.registry.zk.quorum": {
+ "value": "{ZOOKEEPER_QUORUM}",
+ "template": "yes"
+ },
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000",
+ "yarn.node-labels.enabled": false,
+ "yarn.timeline-service.recovery.enabled": "true",
+ "yarn.timeline-service.state-store-class": "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
+ "yarn.timeline-service.leveldb-state-store.path": "/var/log/hadoop-yarn/timeline"
+ },
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.root.default-node-label-expression": "",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {
+ "remove": "yes"
+ },
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {
+ "remove": "yes"
+ }
+ }
+ },
+ "property-mapping": {
+ "nimbus.host": {
+ "map-to": "nimbus.seeds",
+ "coerce-to": "yaml-array"
+ },
+ "mapreduce.job.speculative.speculativecap": {
+ "map-to": "mapreduce.job.speculative.speculative-cap-running-tasks",
+ "from-catalog": "mapred-site",
+ "to-catalog": "mapred-site",
+ "default": "0.1"
+ },
+ "hive.heapsize": {
+ "map-to": "hive.heapsize",
+ "from-catalog": "hive-site",
+ "to-catalog": "hive-env",
+ "required-services": ["HIVE"]
+ }
+ }
+ }
+ ]
+}
[36/55] [abbrv] ambari git commit: AMBARI-18220. Namenode start
failed on moving namenode on a HA cluster. (jamin)
Posted by ol...@apache.org.
AMBARI-18220. Namenode start failed on moving namenode on a HA cluster. (jamin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5eb69463
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5eb69463
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5eb69463
Branch: refs/heads/branch-dev-logsearch
Commit: 5eb69463ed475072ab4a0a038e6f35d5b9b70fe2
Parents: fe768ad
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Sat Aug 20 19:17:44 2016 -0700
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Sat Aug 20 19:21:46 2016 -0700
----------------------------------------------------------------------
.../main/service/reassign/step4_controller.js | 9 ++++++++-
.../views/main/service/reassign/step5_view.js | 2 +-
.../service/reassign/step4_controller_test.js | 21 +++++++++++++-------
3 files changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb69463/ambari-web/app/controllers/main/service/reassign/step4_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/reassign/step4_controller.js b/ambari-web/app/controllers/main/service/reassign/step4_controller.js
index d853039..243fa3b 100644
--- a/ambari-web/app/controllers/main/service/reassign/step4_controller.js
+++ b/ambari-web/app/controllers/main/service/reassign/step4_controller.js
@@ -968,9 +968,16 @@ App.ReassignMasterWizardStep4Controller = App.HighAvailabilityProgressPageContro
this.updateComponent('ZOOKEEPER_SERVER', components.mapProperty('hostName'), "ZOOKEEPER", "Start");
},
+ /**
+ * Start the namenode that is not being moved (applicable only in NameNode HA environment)
+ * @private {startNameNode}
+ */
startNameNode: function () {
var components = this.get('content.masterComponentHosts').filterProperty('component', 'NAMENODE');
- this.updateComponent('NAMENODE', components.mapProperty('hostName').without(this.get('content.reassignHosts.source')), "HDFS", "Start");
+ var sourceHost = this.get('content.reassignHosts.source');
+ var targetHost = this.get('content.reassignHosts.target');
+ var hostname = components.mapProperty('hostName').without(sourceHost).without(targetHost);
+ this.updateComponent('NAMENODE', hostname, "HDFS", "Start");
},
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb69463/ambari-web/app/views/main/service/reassign/step5_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/service/reassign/step5_view.js b/ambari-web/app/views/main/service/reassign/step5_view.js
index 066b23b..c5fc3c8 100644
--- a/ambari-web/app/views/main/service/reassign/step5_view.js
+++ b/ambari-web/app/views/main/service/reassign/step5_view.js
@@ -38,7 +38,7 @@ App.ReassignMasterWizardStep5View = Em.View.extend({
if (this.get('controller.content.reassign.component_name') === 'NAMENODE' && App.get('isHaEnabled')) {
ha = '_ha';
- var nnStartedHost = this.get('controller.content.masterComponentHosts').filterProperty('component', 'NAMENODE').mapProperty('hostName').without(sourceHost);
+ var nnStartedHost = this.get('controller.content.masterComponentHosts').filterProperty('component', 'NAMENODE').mapProperty('hostName').without(sourceHost).without(targetHost);
}
if (this.get('controller.content.reassign.component_name') === 'APP_TIMELINE_SERVER') {
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb69463/ambari-web/test/controllers/main/service/reassign/step4_controller_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/service/reassign/step4_controller_test.js b/ambari-web/test/controllers/main/service/reassign/step4_controller_test.js
index 8898849..5e7b0e3 100644
--- a/ambari-web/test/controllers/main/service/reassign/step4_controller_test.js
+++ b/ambari-web/test/controllers/main/service/reassign/step4_controller_test.js
@@ -932,17 +932,24 @@ describe('App.ReassignMasterWizardStep4Controller', function () {
describe('#startNameNode()', function () {
beforeEach(function () {
sinon.stub(controller, 'updateComponent', Em.K);
- controller.set('content.masterComponentHosts', [{
- component: 'NAMENODE',
- hostName: 'host1'
- }]);
+ controller.set('content.masterComponentHosts', [
+ {
+ component: 'NAMENODE',
+ hostName: 'host1'
+ },
+ {
+ component: 'NAMENODE',
+ hostName: 'host2'
+ }
+ ]);
});
afterEach(function () {
controller.updateComponent.restore();
});
it('reassign host does not match current', function () {
- controller.set('content.reassignHosts.source', 'host2');
+ controller.set('content.reassignHosts.source', 'host3');
+ controller.set('content.reassignHosts.target', 'host2');
controller.startNameNode();
expect(controller.updateComponent.getCall(0).args[0]).to.be.equal('NAMENODE');
expect(controller.updateComponent.getCall(0).args[1][0]).to.be.equal('host1');
@@ -951,9 +958,9 @@ describe('App.ReassignMasterWizardStep4Controller', function () {
});
it('reassign host matches current', function () {
- controller.set('content.reassignHosts.source', 'host1');
+ controller.set('content.reassignHosts.target', 'host2');
controller.startNameNode();
- expect(controller.updateComponent.calledWith('NAMENODE', [], 'HDFS', 'Start')).to.be.true;
+ expect(controller.updateComponent.calledWith('NAMENODE', ['host1'], 'HDFS', 'Start')).to.be.true;
});
});
[51/55] [abbrv] ambari git commit: AMBARI-18214. Restify Log Search
endpoints (oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
deleted file mode 100644
index 0144edc..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java
+++ /dev/null
@@ -1,575 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.rest;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.common.SearchCriteria;
-import org.apache.ambari.logsearch.manager.LogsMgr;
-import org.apache.ambari.logsearch.util.RESTErrorUtil;
-import org.apache.ambari.logsearch.view.VCountList;
-import org.apache.ambari.logsearch.view.VNameValueList;
-import org.apache.ambari.logsearch.view.VNodeList;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.*;
-import static org.apache.ambari.logsearch.doc.DocConstants.ServiceDescriptions.*;
-import static org.apache.ambari.logsearch.doc.DocConstants.ServiceOperationDescriptions.*;
-
-@Api(value = "dashboard", description = "Dashboard operations")
-@Path("dashboard")
-@Component
-@Scope("request")
-public class DashboardREST {
-
- @Autowired
- LogsMgr logMgr;
-
- @Autowired
- RESTErrorUtil restErrorUtil;
-
- @GET
- @Path("/solr/logs_search")
- @Produces({"application/json"})
- @ApiOperation(SEARCH_LOGS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FIND_D, name = "find", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = SOURCE_LOG_ID_D, name = "sourceLogId", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = KEYWORD_TYPE_D, name = "keywordType", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TOKEN_D, name = "token", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = IS_LAST_PAGE_D, name = "isLastPage", dataType = "boolean", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String searchSolrData(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- searchCriteria.addParam("keyword", StringEscapeUtils.unescapeXml(request.getParameter("find")));
- searchCriteria.addParam("sourceLogId", request.getParameter("sourceLogId"));
- searchCriteria.addParam("keywordType",
- request.getParameter("keywordType"));
- searchCriteria.addParam("token",
- request.getParameter("token"));
- searchCriteria.addParam("isLastPage",request.getParameter("isLastPage"));
- return logMgr.searchLogs(searchCriteria);
- }
-
- @GET
- @Path("/hosts")
- @Produces({"application/json"})
- @ApiOperation(GET_HOSTS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query")
- })
- public String getHosts(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addParam("q", request.getParameter("q"));
- return logMgr.getHosts(searchCriteria);
- }
-
- @GET
- @Path("/components")
- @Produces({"application/json"})
- @ApiOperation(GET_COMPONENTS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query")
- })
- public String getComponents(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addParam("q", request.getParameter("q"));
- return logMgr.getComponents(searchCriteria);
- }
-
- @GET
- @Path("/aggregatedData")
- @Produces({"application/json"})
- @ApiOperation(GET_AGGREGATED_INFO_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String getAggregatedInfo(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria();
- searchCriteria.addRequiredServiceLogsParams(request);
- return logMgr.getAggregatedInfo(searchCriteria);
- }
-
- @GET
- @Path("/levels_count")
- @Produces({"application/json"})
- @ApiOperation(GET_LOG_LEVELS_COUNT_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public VCountList getLogLevelsCount(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria();
- searchCriteria.addParam("q", request.getParameter("q"));
- searchCriteria
- .addParam("startDate", request.getParameter("start_time"));
- searchCriteria.addParam("endDate", request.getParameter("end_time"));
- return logMgr.getLogLevelCount(searchCriteria);
- }
-
- @GET
- @Path("/components_count")
- @Produces({"application/json"})
- @ApiOperation(GET_COMPONENTS_COUNT_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public VCountList getComponentsCount(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria();
- searchCriteria.addParam("q", request.getParameter("q"));
- searchCriteria
- .addParam("startDate", request.getParameter("start_time"));
- searchCriteria.addParam("endDate", request.getParameter("end_time"));
- return logMgr.getComponentsCount(searchCriteria);
- }
-
- @GET
- @Path("/hosts_count")
- @Produces({"application/json"})
- @ApiOperation(GET_HOSTS_COUNT_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query")
- })
- public VCountList getHostsCount(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria();
- searchCriteria.addParam("q", request.getParameter("q"));
- searchCriteria
- .addParam("startDate", request.getParameter("start_time"));
- searchCriteria.addParam("endDate", request.getParameter("end_time"));
- searchCriteria.addParam("excludeQuery", StringEscapeUtils
- .unescapeXml(request.getParameter("excludeQuery")));
- searchCriteria.addParam("includeQuery", StringEscapeUtils
- .unescapeXml(request.getParameter("includeQuery")));
- return logMgr.getHostsCount(searchCriteria);
- }
-
- @GET
- @Path("/getTreeExtension")
- @Produces({"application/json"})
- @ApiOperation(GET_TREE_EXTENSION_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public VNodeList getTreeExtension(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- searchCriteria.addParam("hostName", request.getParameter("hostName"));
- return logMgr.getTreeExtension(searchCriteria);
- }
-
- @GET
- @Path("/getLogLevelCounts")
- @Produces({"application/json"})
- @ApiOperation(GET_LOG_LEVELS_COUNT_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public VNameValueList getLogsLevelCount(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- return logMgr.getLogsLevelCount(searchCriteria);
- }
-
- @GET
- @Path("/getHistogramData")
- @Produces({"application/json"})
- @ApiOperation(GET_HISTOGRAM_DATA_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String getHistogramData(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- searchCriteria.addParam("unit", request.getParameter("unit"));
- return logMgr.getHistogramData(searchCriteria);
- }
-
- @GET
- @Path("/cancelFindRequest")
- @Produces({"application/json"})
- @ApiOperation(CANCEL_FIND_REQUEST_OD)
- public String cancelFindRequest(@Context HttpServletRequest request) {
- return logMgr.cancelFindRequestByDate(request);
- }
-
- @GET
- @Path("/exportToTextFile")
- @Produces({"application/json"})
- @ApiOperation(EXPORT_TO_TEXT_FILE_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FORMAT_D, name = "format", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UTC_OFFSET_D, name = "utcOffset", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public Response exportToTextFile(@Context HttpServletRequest request) {
-
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- searchCriteria.addParam("format", request.getParameter("format"));
- searchCriteria.addParam("utcOffset", request.getParameter("utcOffset"));
- return logMgr.exportToTextFile(searchCriteria);
-
- }
-
- @GET
- @Path("/getHostListByComponent")
- @Produces({"application/json"})
- @ApiOperation(GET_HOST_LIST_BY_COMPONENT_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String getHostListByComponent(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- searchCriteria.addParam("componentName",
- request.getParameter("componentName"));
- return logMgr.getHostListByComponent(searchCriteria);
- }
-
- @GET
- @Path("/getComponentListWithLevelCounts")
- @Produces({"application/json"})
- @ApiOperation(GET_COMPONENT_LIST_WITH_LEVEL_COUNT_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String getComponentListWithLevelCounts(
- @Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- return logMgr.getComponentListWithLevelCounts(searchCriteria);
- }
-
- @GET
- @Path("/solr/getBundleIdBoundaryDates")
- @Produces({"application/json"})
- @ApiOperation(GET_EXTREME_DATES_FOR_BUNDLE_ID_OD)
- public String getExtremeDatesForBundelId(@Context HttpServletRequest request) {
-
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addParam(LogSearchConstants.BUNDLE_ID,
- request.getParameter("bundle_id"));
-
- return logMgr.getExtremeDatesForBundelId(searchCriteria);
-
- }
-
- @GET
- @Path("/getServiceLogsFieldsName")
- @Produces({"application/json"})
- @ApiOperation(GET_SERVICE_LOGS_FIELD_NAME_OD)
- public String getServiceLogsFieldsName() {
- return logMgr.getServiceLogsFieldsName();
- }
-
- @GET
- @Path("/getServiceLogsSchemaFieldsName")
- @Produces({"application/json"})
- @ApiOperation(GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD)
- public String getServiceLogsSchemaFieldsName() {
- return logMgr.getServiceLogsSchemaFieldsName();
- }
-
- @GET
- @Path("/getAnyGraphData")
- @Produces({"application/json"})
- @ApiOperation(GET_ANY_GRAPH_DATA_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = X_AXIS_D, name = "xAxis", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = Y_AXIS_D, name = "yAxis", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = STACK_BY_D, name = "stackBy", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String getAnyGraphData(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("xAxis", request.getParameter("xAxis"));
- searchCriteria.addParam("yAxis", request.getParameter("yAxis"));
- searchCriteria.addParam("stackBy", request.getParameter("stackBy"));
- searchCriteria.addParam("from", request.getParameter("from"));
- searchCriteria.addParam("to", request.getParameter("to"));
- searchCriteria.addParam("unit", request.getParameter("unit"));
- return logMgr.getAnyGraphData(searchCriteria);
- }
-
- @GET
- @Path("/getAfterBeforeLogs")
- @Produces({"application/json"})
- @ApiOperation(GET_AFTER_BEFORE_LOGS_OD)
- @ApiImplicitParams(value = {
- @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_D,name = "component", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ID_D, name = "id", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = SCROLL_TYPE_D, name = "scrollType", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = NUMBER_ROWS_D, name = "numberRows", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
- })
- public String getAfterBeforeLogs(@Context HttpServletRequest request) {
- SearchCriteria searchCriteria = new SearchCriteria(request);
- searchCriteria.addRequiredServiceLogsParams(request);
- searchCriteria.addParam("hostLogFile", request.getParameter("host"));
- searchCriteria.addParam("compLogFile",
- request.getParameter("component"));
- searchCriteria.addParam("id", request.getParameter("id"));
- searchCriteria.addParam("scrollType",
- request.getParameter("scrollType"));
- searchCriteria.addParam("numberRows",
- request.getParameter("numberRows"));
- return logMgr.getAfterBeforeLogs(searchCriteria);
- }
-
- @GET
- @Path("/getHadoopServiceConfigJSON")
- @Produces({"application/json"})
- @ApiOperation(GET_HADOOP_SERVICE_CONFIG_JSON_OD)
- public String getHadoopServiceConfigJSON() {
- return logMgr.getHadoopServiceConfigJSON();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
index d53cff9..6099e0f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/LogFileREST.java
@@ -63,13 +63,14 @@ public class LogFileREST {
}
@GET
- @Path("/getLogFileTail")
+ @Path("/tail")
@Produces({"application/json"})
@ApiOperation(GET_LOG_FILE_TAIL_OD)
@ApiImplicitParams(value = {
@ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
@ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
- @ApiImplicitParam(value = LOG_TYPE_D, name = "logType", dataType = "string", paramType = "query")
+ @ApiImplicitParam(value = LOG_TYPE_D, name = "logType", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TAIL_SIZE_D, name = "tailSize", dataType = "string", paramType = "query")
})
public String getLogFileTail(@Context HttpServletRequest request) {
SearchCriteria searchCriteria = new SearchCriteria();
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
index af48acd..5218f5d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/PublicREST.java
@@ -40,7 +40,7 @@ public class PublicREST {
PublicMgr generalMgr;
@GET
- @Path("/getGeneralConfig")
+ @Path("/config")
@ApiOperation(OBTAIN_GENERAL_CONFIG_OD)
public String getGeneralConfig() {
return generalMgr.getGeneralConfig();
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsREST.java
new file mode 100644
index 0000000..373f7ff
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsREST.java
@@ -0,0 +1,578 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.rest;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.manager.LogsMgr;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.ambari.logsearch.view.VCountList;
+import org.apache.ambari.logsearch.view.VNameValueList;
+import org.apache.ambari.logsearch.view.VNodeList;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.ServiceDescriptions.*;
+import static org.apache.ambari.logsearch.doc.DocConstants.ServiceOperationDescriptions.*;
+
+@Api(value = "service/logs", description = "Service log operations")
+@Path("service/logs")
+@Component
+@Scope("request")
+public class ServiceLogsREST {
+
+ @Autowired
+ LogsMgr logMgr;
+
+ @Autowired
+ RESTErrorUtil restErrorUtil;
+
+ @GET
+ @Produces({"application/json"})
+ @ApiOperation(SEARCH_LOGS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FIND_D, name = "find", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = SOURCE_LOG_ID_D, name = "sourceLogId", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = KEYWORD_TYPE_D, name = "keywordType", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TOKEN_D, name = "token", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = IS_LAST_PAGE_D, name = "isLastPage", dataType = "boolean", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String searchSolrData(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ searchCriteria.addParam("keyword", StringEscapeUtils.unescapeXml(request.getParameter("find")));
+ searchCriteria.addParam("sourceLogId", request.getParameter("sourceLogId"));
+ searchCriteria.addParam("keywordType",
+ request.getParameter("keywordType"));
+ searchCriteria.addParam("token",
+ request.getParameter("token"));
+ searchCriteria.addParam("isLastPage",request.getParameter("isLastPage"));
+ return logMgr.searchLogs(searchCriteria);
+ }
+
+ @GET
+ @Path("/hosts")
+ @Produces({"application/json"})
+ @ApiOperation(GET_HOSTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query")
+ })
+ public String getHosts(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addParam("q", request.getParameter("q"));
+ return logMgr.getHosts(searchCriteria);
+ }
+
+ @GET
+ @Path("/components")
+ @Produces({"application/json"})
+ @ApiOperation(GET_COMPONENTS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query")
+ })
+ public String getComponents(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addParam("q", request.getParameter("q"));
+ return logMgr.getComponents(searchCriteria);
+ }
+
+ @GET
+ @Path("/aggregated")
+ @Produces({"application/json"})
+ @ApiOperation(GET_AGGREGATED_INFO_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String getAggregatedInfo(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.addRequiredServiceLogsParams(request);
+ return logMgr.getAggregatedInfo(searchCriteria);
+ }
+
+ @GET
+ @Path("/levels/count")
+ @Produces({"application/json"})
+ @ApiOperation(GET_LOG_LEVELS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public VCountList getLogLevelsCount(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.addParam("q", request.getParameter("q"));
+ searchCriteria
+ .addParam("startDate", request.getParameter("start_time"));
+ searchCriteria.addParam("endDate", request.getParameter("end_time"));
+ return logMgr.getLogLevelCount(searchCriteria);
+ }
+
+ @GET
+ @Path("/components/count")
+ @Produces({"application/json"})
+ @ApiOperation(GET_COMPONENTS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public VCountList getComponentsCount(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.addParam("q", request.getParameter("q"));
+ searchCriteria
+ .addParam("startDate", request.getParameter("start_time"));
+ searchCriteria.addParam("endDate", request.getParameter("end_time"));
+ return logMgr.getComponentsCount(searchCriteria);
+ }
+
+ @GET
+ @Path("/hosts/count")
+ @Produces({"application/json"})
+ @ApiOperation(GET_HOSTS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = QUERY_D, name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query")
+ })
+ public VCountList getHostsCount(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.addParam("q", request.getParameter("q"));
+ searchCriteria
+ .addParam("startDate", request.getParameter("start_time"));
+ searchCriteria.addParam("endDate", request.getParameter("end_time"));
+ searchCriteria.addParam("excludeQuery", StringEscapeUtils
+ .unescapeXml(request.getParameter("excludeQuery")));
+ searchCriteria.addParam("includeQuery", StringEscapeUtils
+ .unescapeXml(request.getParameter("includeQuery")));
+ return logMgr.getHostsCount(searchCriteria);
+ }
+
+ @GET
+ @Path("/tree")
+ @Produces({"application/json"})
+ @ApiOperation(GET_TREE_EXTENSION_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public VNodeList getTreeExtension(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ searchCriteria.addParam("hostName", request.getParameter("hostName"));
+ return logMgr.getTreeExtension(searchCriteria);
+ }
+
+ @GET
+ @Path("/levels/counts/namevalues")
+ @Produces({"application/json"})
+ @ApiOperation(GET_LOG_LEVELS_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public VNameValueList getLogsLevelCount(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ return logMgr.getLogsLevelCount(searchCriteria);
+ }
+
+ @GET
+ @Path("/histogram")
+ @Produces({"application/json"})
+ @ApiOperation(GET_HISTOGRAM_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String getHistogramData(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ searchCriteria.addParam("unit", request.getParameter("unit"));
+ return logMgr.getHistogramData(searchCriteria);
+ }
+
+ @GET
+ @Path("/request/cancel")
+ @Produces({"application/json"})
+ @ApiOperation(CANCEL_FIND_REQUEST_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = TOKEN_D, name = "token", dataType = "string", paramType = "query"),
+ })
+ public String cancelFindRequest(@Context HttpServletRequest request) {
+ String uniqueId = request.getParameter("token");
+ return logMgr.cancelFindRequestByDate(uniqueId);
+ }
+
+ @GET
+ @Path("/export")
+ @Produces({"application/json"})
+ @ApiOperation(EXPORT_TO_TEXT_FILE_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FORMAT_D, name = "format", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UTC_OFFSET_D, name = "utcOffset", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public Response exportToTextFile(@Context HttpServletRequest request) {
+
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ searchCriteria.addParam("format", request.getParameter("format"));
+ searchCriteria.addParam("utcOffset", request.getParameter("utcOffset"));
+ return logMgr.exportToTextFile(searchCriteria);
+
+ }
+
+ @GET
+ @Path("/hosts/components")
+ @Produces({"application/json"})
+ @ApiOperation(GET_HOST_LIST_BY_COMPONENT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String getHostListByComponent(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ searchCriteria.addParam("componentName",
+ request.getParameter("componentName"));
+ return logMgr.getHostListByComponent(searchCriteria);
+ }
+
+ @GET
+ @Path("/components/level/counts")
+ @Produces({"application/json"})
+ @ApiOperation(GET_COMPONENT_LIST_WITH_LEVEL_COUNT_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D, name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String getComponentListWithLevelCounts(
+ @Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ return logMgr.getComponentListWithLevelCounts(searchCriteria);
+ }
+
+ @GET
+ @Path("/solr/boundarydates")
+ @Produces({"application/json"})
+ @ApiOperation(GET_EXTREME_DATES_FOR_BUNDLE_ID_OD)
+ public String getExtremeDatesForBundelId(@Context HttpServletRequest request) {
+
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addParam(LogSearchConstants.BUNDLE_ID,
+ request.getParameter("bundle_id"));
+
+ return logMgr.getExtremeDatesForBundelId(searchCriteria);
+
+ }
+
+ @GET
+ @Path("/fields")
+ @Produces({"application/json"})
+ @ApiOperation(GET_SERVICE_LOGS_FIELD_NAME_OD)
+ public String getServiceLogsFieldsName() {
+ return logMgr.getServiceLogsFieldsName();
+ }
+
+ @GET
+ @Path("/schema/fields")
+ @Produces({"application/json"})
+ @ApiOperation(GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD)
+ public String getServiceLogsSchemaFieldsName() {
+ return logMgr.getServiceLogsSchemaFieldsName();
+ }
+
+ @GET
+ @Path("/anygraph")
+ @Produces({"application/json"})
+ @ApiOperation(GET_ANY_GRAPH_DATA_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = X_AXIS_D, name = "xAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = Y_AXIS_D, name = "yAxis", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = STACK_BY_D, name = "stackBy", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FROM_D, name = "from", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TO_D, name = "to", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = UNIT_D, name = "unit", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String getAnyGraphData(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("xAxis", request.getParameter("xAxis"));
+ searchCriteria.addParam("yAxis", request.getParameter("yAxis"));
+ searchCriteria.addParam("stackBy", request.getParameter("stackBy"));
+ searchCriteria.addParam("from", request.getParameter("from"));
+ searchCriteria.addParam("to", request.getParameter("to"));
+ searchCriteria.addParam("unit", request.getParameter("unit"));
+ return logMgr.getAnyGraphData(searchCriteria);
+ }
+
+ @GET
+ @Path("/truncated")
+ @Produces({"application/json"})
+ @ApiOperation(GET_AFTER_BEFORE_LOGS_OD)
+ @ApiImplicitParams(value = {
+ @ApiImplicitParam(value = HOST_D, name = "host", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_D,name = "component", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ID_D, name = "id", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = SCROLL_TYPE_D, name = "scrollType", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = NUMBER_ROWS_D, name = "numberRows", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = ADVANCED_SEARCH_D, name = "advancedSearch", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = QUERY_D,name = "q", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = LEVEL_D, name = "level", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = TREE_PARAMS_D, name = "treeParams", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COLUMN_QUERY_D, name = "columnQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = I_MESSAGE_D, name = "iMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_E_MESSAGE_D, name = "gEMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = E_MESSAGE_D, name = "eMessage", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = G_MUST_NOT_D, name = "gMustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_BE_D, name = "mustBe", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = MUST_NOT_D, name = "mustNot", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = HOST_NAME_D, name = "host_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = COMPONENT_NAME_D, name = "component_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = FILE_NAME_D, name = "file_name", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = EXCLUDE_QUERY_D, name = "excludeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = INCLUDE_QUERY_D, name = "includeQuery", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = START_TIME_D, name = "start_time", dataType = "string", paramType = "query"),
+ @ApiImplicitParam(value = END_TIME_D, name = "end_time", dataType = "string", paramType = "query")
+ })
+ public String getAfterBeforeLogs(@Context HttpServletRequest request) {
+ SearchCriteria searchCriteria = new SearchCriteria(request);
+ searchCriteria.addRequiredServiceLogsParams(request);
+ searchCriteria.addParam("hostLogFile", request.getParameter("host"));
+ searchCriteria.addParam("compLogFile",
+ request.getParameter("component"));
+ searchCriteria.addParam("id", request.getParameter("id"));
+ searchCriteria.addParam("scrollType",
+ request.getParameter("scrollType"));
+ searchCriteria.addParam("numberRows",
+ request.getParameter("numberRows"));
+ return logMgr.getAfterBeforeLogs(searchCriteria);
+ }
+
+ @GET
+ @Path("/serviceconfig")
+ @Produces({"application/json"})
+ @ApiOperation(GET_HADOOP_SERVICE_CONFIG_JSON_OD)
+ public String getHadoopServiceConfigJSON() {
+ return logMgr.getHadoopServiceConfigJSON();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
index 4b1675f..a79a737 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/UserConfigREST.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -58,7 +58,6 @@ public class UserConfigREST {
UserConfigMgr userConfigMgr;
@POST
- @Path("/saveUserConfig")
@Produces({"application/json"})
@ApiOperation(SAVE_USER_CONFIG_OD)
public String saveUserConfig(VUserConfig vhist) {
@@ -66,7 +65,6 @@ public class UserConfigREST {
}
@PUT
- @Path("/updateUserConfig")
@Produces({"application/json"})
@ApiOperation(UPDATE_USER_CONFIG_OD)
public String updateUserConfig(VUserConfig vhist) {
@@ -74,14 +72,13 @@ public class UserConfigREST {
}
@DELETE
- @Path("/deleteUserConfig/{id}")
+ @Path("/{id}")
@ApiOperation(DELETE_USER_CONFIG_OD)
public void deleteUserConfig(@PathParam("id") String id) {
userConfigMgr.deleteUserConfig(id);
}
@GET
- @Path("/getUserConfig")
@Produces({"application/json"})
@ApiOperation(GET_USER_CONFIG_OD)
@ApiImplicitParams(value = {
@@ -101,7 +98,7 @@ public class UserConfigREST {
}
@GET
- @Path("/user_filter")
+ @Path("/users/filter")
@Produces({"application/json"})
@ApiOperation(GET_USER_FILTER_OD)
public String getUserFilter(@Context HttpServletRequest request) {
@@ -109,7 +106,7 @@ public class UserConfigREST {
}
@POST
- @Path("/user_filter")
+ @Path("/users/filter")
@Produces({"application/json"})
@ApiOperation(UPDATE_USER_FILTER_OD)
public String createUserFilter(String json) {
@@ -117,7 +114,7 @@ public class UserConfigREST {
}
@PUT
- @Path("/user_filter/{id}")
+ @Path("/users/filter/{id}")
@Produces({"application/json"})
@ApiOperation(GET_USER_FILTER_BY_ID_OD)
public String updateUserFilter(String json) {
@@ -125,7 +122,7 @@ public class UserConfigREST {
}
@GET
- @Path("/getAllUserName")
+ @Path("/users/names")
@Produces({"application/json"})
@ApiOperation(GET_ALL_USER_NAMES_OD)
public String getAllUserName() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
index 44f1aeb..038cec2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/login.html
@@ -1,4 +1,4 @@
-<!--
+<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
@@ -35,27 +35,27 @@
var passDiv = $('#passCont');
$.ajax({
- url : "/api/v1/public/getGeneralConfig",
+ url : "/api/v1/public/config",
type: "GET",
async: true,
dataType: 'json',
- success:function(results,status)
+ success:function(results,status)
{
for(var i in results.vNameValues){
if ((results.vNameValues[i].name === "simpleAuth") && (results.vNameValues[i].value === "true")){
$("#password").val("");
passDiv.hide();
- }
+ }
if (passDiv.is(':hidden'))
$("#password").prop('required',false);
}
},
- error: function(errorThrown)
+ error: function(errorThrown)
{
passDiv.show();
},
complete : function(){
- $('[data-id="loader"]').hide();
+ $('[data-id="loader"]').hide();
}
});
});
@@ -105,7 +105,7 @@
</div>
<div data-id="loader" class="loading"></div>
</div>
- </div>
+ </div>
</div>
<script type="text/javascript">
@@ -116,20 +116,20 @@
var postData = {};
var formURL = ($(this).attr("action")) ? $(this).attr("action") : "/login";
postData = {"username" : $("#username").val() , "password" : $("#password").val()};
-
+
$.ajax({
url : formURL,
type: "POST",
data : postData,
- success:function(results,status)
+ success:function(results,status)
{
window.location = 'index.html'+window.location.search;
},
- error: function(errorThrown)
+ error: function(errorThrown)
{
showError(errorThrown);
}
- });
+ });
return false;
});
@@ -139,11 +139,11 @@
function showError(errorThrown){
var errorMsg = errorThrown.status;
-
+
switch(errorMsg){
- case 401: $('.errorBox').show();
+ case 401: $('.errorBox').show();
break;
-
+
default: $('.errorBox').hide();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VAuditLogListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VAuditLogListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VAuditLogListBase.js
index 12f7c31..0f72697 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VAuditLogListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VAuditLogListBase.js
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,18 +17,18 @@
* under the License.
*/
-
+
define(['require',
'collections/BaseCollection',
'utils/Globals',
'models/VAuditLog'
],function(require,BaseCollection,Globals,VAuditLog){
- 'use strict';
+ 'use strict';
var VAuditLogListBase = BaseCollection.extend(
/** @lends VAuditLogListBase.prototype */
{
- url: Globals.baseURL + 'audit/getAuditLogs',
+ url: Globals.baseURL + 'audit/logs',
model : VAuditLog,
@@ -49,8 +49,8 @@ define(['require',
*************************/
getAuditSchemaFieldsName : function(token, options){
- var url = Globals.baseURL + 'audit/getAuditSchemaFieldsName';
-
+ var url = Globals.baseURL + 'audit/logs/schema/fields';
+
options = _.extend({
data : $.param(token),
contentType : 'application/json',
@@ -71,5 +71,3 @@ define(['require',
return VAuditLogListBase;
});
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VEventHistoryListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VEventHistoryListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VEventHistoryListBase.js
index f6f720d..06b2619 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VEventHistoryListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VEventHistoryListBase.js
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,11 +24,11 @@ define(['require',
'models/VEventHistory'
],function(require,BaseCollection,Globals,VEventHistory) {
'use strict';
-
+
var VEventHistoryListBase = BaseCollection.extend(
/** @lends VEventHistoryListBase.prototype */
{
- url: Globals.baseURL + 'userconfig/getUserConfig',
+ url: Globals.baseURL + 'userconfig',
model: VEventHistory,
@@ -48,7 +48,7 @@ define(['require',
*************************/
saveEventHistory: function(postData, options) {
- var url = Globals.baseURL + 'userconfig/saveUserConfig';
+ var url = Globals.baseURL + 'userconfig';
options = _.extend({
data: JSON.stringify(postData),
@@ -59,7 +59,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'POST', options);
},
saveDashboard: function(postData, options) {
- var url = Globals.baseURL + 'userconfig/saveUserConfig';
+ var url = Globals.baseURL + 'userconfig';
options = _.extend({
data: JSON.stringify(postData),
@@ -70,7 +70,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'POST', options);
},
deleteEventHistory: function(postData, options) {
- var url = Globals.baseURL + 'userconfig/deleteUserConfig/' + postData.id;
+ var url = Globals.baseURL + 'userconfig/' + postData.id;
options = _.extend({
contentType: 'application/json',
[18/55] [abbrv] ambari git commit: AMBARI-18142. Define
keytab/principal for Spark Thrift Server - addendum patch (Jianfeng Zhang via
smohanty)
Posted by ol...@apache.org.
AMBARI-18142. Define keytab/principal for Spark Thrift Server - addendum patch (Jianfeng Zhang via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a57daad4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a57daad4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a57daad4
Branch: refs/heads/branch-dev-logsearch
Commit: a57daad47194438f5df10414130aa18effc3446e
Parents: 0e5fb8e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 18 13:29:19 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 18 13:29:19 2016 -0700
----------------------------------------------------------------------
.../SPARK/1.2.1/package/scripts/spark_service.py | 5 +++++
.../SPARK2/2.0.0/package/scripts/spark_service.py | 5 +++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a57daad4/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
index d2f5ef9..82a010a 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
@@ -76,6 +76,11 @@ def spark_service(name, upgrade_type=None, action=None):
raise
elif name == 'sparkthriftserver':
+ if params.security_enabled:
+ hive_principal = params.hive_kerberos_principal
+ hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ")
+ Execute(hive_kinit_cmd, user=params.hive_user)
+
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
try:
http://git-wip-us.apache.org/repos/asf/ambari/blob/a57daad4/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
index 308dd3d..590b299 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
@@ -96,6 +96,11 @@ def spark_service(name, upgrade_type=None, action=None):
raise
elif name == 'sparkthriftserver':
+ if params.security_enabled:
+ hive_principal = params.hive_kerberos_principal
+ hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ")
+ Execute(hive_kinit_cmd, user=params.hive_user)
+
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
try:
[05/55] [abbrv] ambari git commit: Revert "AMBARI-18156 Remove
upgradeHelper.py and corresponding files due to drop of the manual upgrade
(dsen)"
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c758b7ff/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json
new file mode 100644
index 0000000..50c63de
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.2.2.json
@@ -0,0 +1,465 @@
+{
+ "version": "1.0",
+ "stacks": [
+ {
+ "name": "HDP",
+ "old-version": "2.1",
+ "target-version": "2.2.2",
+ "options": {
+ "config-types": {
+ "capacity-scheduler": {
+ "merged-copy": "yes"
+ },
+ "cluster-env": {
+ "merged-copy": "yes"
+ },
+ "core-site": {
+ "merged-copy": "yes"
+ },
+ "falcon-startup.properties": {
+ "merged-copy": "yes"
+ },
+ "hadoop-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-env": {
+ "merged-copy": "yes"
+ },
+ "hbase-site": {
+ "merged-copy": "yes"
+ },
+ "hdfs-log4j": {
+ "merged-copy": "yes"
+ },
+ "hdfs-site": {
+ "merged-copy": "yes"
+ },
+ "hive-env": {
+ "merged-copy": "yes"
+ },
+ "hive-site": {
+ "merged-copy": "yes"
+ },
+ "mapred-env": {
+ "merged-copy": "yes"
+ },
+ "mapred-site": {
+ "merged-copy": "yes"
+ },
+ "oozie-env": {
+ "merged-copy": "yes"
+ },
+ "oozie-site": {
+ "merged-copy": "yes"
+ },
+ "storm-env": {
+ "merged-copy": "yes"
+ },
+ "storm-site": {
+ "merged-copy": "yes"
+ },
+ "tez-site": {
+ "merged-copy": "yes"
+ },
+ "webhcat-site": {
+ "merged-copy": "yes"
+ },
+ "yarn-site": {
+ "merged-copy": "yes"
+ }
+ }
+ },
+ "properties": {
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+ "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity": {"remove": "yes"},
+ "yarn.scheduler.capacity.root.default-node-label-expression": " "
+ },
+ "core-site": {
+ "hadoop.http.authentication.simple.anonymous.allowed": "true"
+ },
+ "falcon-startup.properties": {
+ "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService,\\\n org.apache.falcon.workflow.WorkflowJobEndNotificationService, \\\n org.apache.falcon.service.ProcessSubscriberService,\\\n org.apache.falcon.entity.store.ConfigurationStore,\\\n org.apache.falcon.rerun.service.RetryService,\\\n org.apache.falcon.rerun.service.LateRunService,\\\n org.apache.falcon.service.LogCleanupService,\\\n org.apache.falcon.metadata.MetadataMappingService",
+ "*.falcon.enableTLS": "false",
+ "*.falcon.graph.blueprints.graph": "com.thinkaurelius.titan.core.TitanFactory",
+ "*.falcon.graph.storage.backend": "berkeleyje",
+ "*.falcon.security.authorization.admin.groups": "falcon",
+ "*.falcon.security.authorization.admin.users": "falcon,ambari-qa",
+ "*.falcon.security.authorization.enabled": "false",
+ "*.falcon.security.authorization.provider": "org.apache.falcon.security.DefaultAuthorizationProvider",
+ "*.falcon.security.authorization.superusergroup": "falcon",
+ "*.journal.impl": "org.apache.falcon.transaction.SharedFileSystemJournal",
+ "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
+ "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap"
+ },
+ "hadoop-env": {
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appended to HAD
OOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{
{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The following applies
to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HADOOP_MAST
ER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /usr/share/java/*my
sql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n if [ -d \"/etc/tez/conf/\" ]; then\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n fi\nfi\n\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\""
+ },
+ "hbase-env": {
+ "content": "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to enable JMX expo
rting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{log_dir}}
\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport HBASE_REGIONSERVER_OPTS=
\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}"
+ },
+ "hbase-site": {
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hstore.flush.retries.number": {"remove": "yes"}
+ },
+ "hdfs-log4j": {
+ "content": "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.logger=INF
O,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console
.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize=256MB\nhadoop.security.log.maxbackupindex=20\nlog4j.category.SecurityLogger=${hadoop.security.logger}\nhad
oop.security.log.file=SecurityAuth.audit\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\nl
og4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n#
Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize=256MB\nlog4j.appender.RFA.MaxBackupIndex=10\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.ap
pender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN"
+ },
+ "hdfs-site": {
+ "dfs.datanode.max.transfer.threads": "16384",
+ "dfs.namenode.startup.delay.block.deletion.sec": "3600"
+ },
+ "hive-env": {
+ "content": "\n if [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\n fi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_config_dir}}\n\n# Folder containing extra libraries required for hive compilat
ion/execution can be controlled by:\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\nelif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog\nfi\n\nexport METASTORE_PORT={{hive_metastore_port}}"
+ },
+ "hive-site": {
+ "hive.auto.convert.sortmerge.join.to.mapjoin": "false",
+ "hive.cbo.enable": "true",
+ "hive.cli.print.header": "false",
+ "hive.cluster.delegation.token.store.class": "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
+ "hive.cluster.delegation.token.store.zookeeper.znode": "/hive/cluster/delegation",
+ "hive.conf.restricted.list": "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
+ "hive.convert.join.bucket.mapjoin.tez": "false",
+ "hive.exec.compress.intermediate": "false",
+ "hive.exec.compress.output": "false",
+ "hive.exec.dynamic.partition": "true",
+ "hive.exec.dynamic.partition.mode": "nonstrict",
+ "hive.exec.max.created.files": "100000",
+ "hive.exec.max.dynamic.partitions": "5000",
+ "hive.exec.max.dynamic.partitions.pernode": "2000",
+ "hive.exec.orc.compression.strategy": "SPEED",
+ "hive.exec.orc.default.compress": "ZLIB",
+ "hive.exec.orc.default.stripe.size": "67108864",
+ "hive.exec.parallel": "false",
+ "hive.exec.parallel.thread.number": "8",
+ "hive.exec.reducers.bytes.per.reducer": "67108864",
+ "hive.exec.reducers.max": "1009",
+ "hive.exec.scratchdir": "/tmp/hive",
+ "hive.exec.submit.local.task.via.child": "true",
+ "hive.exec.submitviachild": "false",
+ "hive.fetch.task.aggr": "false",
+ "hive.fetch.task.conversion": "more",
+ "hive.fetch.task.conversion.threshold": "1073741824",
+ "hive.map.aggr.hash.force.flush.memory.threshold": "0.9",
+ "hive.map.aggr.hash.min.reduction": "0.5",
+ "hive.map.aggr.hash.percentmemory": "0.5",
+ "hive.mapjoin.optimized.hashtable": "true",
+ "hive.merge.mapfiles": "true",
+ "hive.merge.mapredfiles": "false",
+ "hive.merge.orcfile.stripe.level": "true",
+ "hive.merge.rcfile.block.level": "true",
+ "hive.merge.size.per.task": "256000000",
+ "hive.merge.smallfiles.avgsize": "16000000",
+ "hive.merge.tezfiles": "false",
+ "hive.metastore.authorization.storage.checks": "false",
+ "hive.metastore.client.connect.retry.delay": "5s",
+ "hive.metastore.client.socket.timeout": "1800s",
+ "hive.metastore.connect.retries": "24",
+ "hive.metastore.failure.retries": "24",
+ "hive.metastore.server.max.threads": "100000",
+ "hive.optimize.constant.propagation": "true",
+ "hive.optimize.metadataonly": "true",
+ "hive.optimize.null.scan": "true",
+ "hive.optimize.sort.dynamic.partition": "false",
+ "hive.orc.compute.splits.num.threads": "10",
+ "hive.prewarm.enabled": "false",
+ "hive.prewarm.numcontainers": "10",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
+ "hive.security.metastore.authenticator.manager": "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "hive.security.metastore.authorization.auth.reads": "true",
+ "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly",
+ "hive.server2.allow.user.substitution": "true",
+ "hive.server2.logging.operation.enabled": "true",
+ "hive.server2.logging.operation.log.location": "/tmp/hive/operation_logs",
+ "hive.server2.table.type.mapping": "CLASSIC",
+ "hive.server2.thrift.http.path": "cliservice",
+ "hive.server2.thrift.http.port": "10001",
+ "hive.server2.thrift.max.worker.threads": "500",
+ "hive.server2.thrift.sasl.qop": "auth",
+ "hive.server2.use.SSL": "false",
+ "hive.smbjoin.cache.rows": "10000",
+ "hive.stats.dbclass": "fs",
+ "hive.stats.fetch.column.stats": "false",
+ "hive.stats.fetch.partition.stats": "true",
+ "hive.support.concurrency": "false",
+ "hive.tez.auto.reducer.parallelism": "false",
+ "hive.tez.cpu.vcores": "-1",
+ "hive.tez.dynamic.partition.pruning": "true",
+ "hive.tez.dynamic.partition.pruning.max.data.size": "104857600",
+ "hive.tez.dynamic.partition.pruning.max.event.size": "1048576",
+ "hive.tez.log.level": "INFO",
+ "hive.tez.max.partition.factor": "2.0",
+ "hive.tez.min.partition.factor": "0.25",
+ "hive.tez.smb.number.waves": "0.5",
+ "hive.user.install.directory": "/user/",
+ "hive.vectorized.execution.reduce.enabled": "false",
+ "hive.vectorized.groupby.checkinterval": "4096",
+ "hive.zookeeper.client.port": "2181",
+ "hive.zookeeper.namespace": "hive_zookeeper_namespace",
+ "fs.file.impl.disable.cache": {"remove": "yes"},
+ "fs.hdfs.impl.disable.cache": {"remove": "yes"},
+ "hive.auto.convert.sortmerge.join.noconditionaltask": {"remove": "yes"},
+ "hive.heapsize": {"remove": "yes"},
+ "hive.optimize.mapjoin.mapreduce": {"remove": "yes"},
+ "hive.server2.enable.impersonation": {"remove": "yes"}
+ },
+ "mapred-env": {
+ "content": "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n#export HADOOP_JOB_HISTORYSERVER_OPTS=\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\""
+ },
+ "mapred-site": {
+ "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "mapreduce.admin.user.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "mapreduce.application.classpath": "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "mapreduce.application.framework.path": "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
+ "mapreduce.job.emit-timeline-data": "false",
+ "mapreduce.jobhistory.bind-host": "0.0.0.0",
+ "mapreduce.reduce.shuffle.fetch.retry.enabled": "1",
+ "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000",
+ "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000",
+ "yarn.app.mapreduce.am.admin-command-opts": "-Dhdp.version=${hdp.version}",
+ "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}"
+ },
+ "oozie-env": {
+ "content": "\n#!/bin/bash\n\nif [ -d \"/usr/lib/bigtop-tomcat\" ]; then\n export OOZIE_CONFIG=${OOZIE_CONFIG:-/etc/oozie/conf}\n export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}\n export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}\n export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat\nfi\n\n#Set JAVA HOME\nexport JAVA_HOME={{java_home}}\n\nexport JRE_HOME=${JAVA_HOME}\n\n# Set Oozie specific environment variables here.\n\n# Settings for the Embedded Tomcat that runs Oozie\n# Java System properties for Oozie should be specified in this variable\n#\n# export CATALINA_OPTS=\n\n# Oozie configuration file to load from Oozie configuration directory\n#\n# export OOZIE_CONFIG_FILE=oozie-site.xml\n\n# Oozie logs directory\n#\nexport OOZIE_LOG={{oozie_log_dir}}\n\n# Oozie pid directory\n#\nexport CATALINA_PID={{pid_file}}\n\n#Location of the data for oozie\nexport OOZIE_DATA={{oozie_data_dir}}\n\n# Oozie Log4J configuration file to load from Oozie configuration directo
ry\n#\n# export OOZIE_LOG4J_FILE=oozie-log4j.properties\n\n# Reload interval of the Log4J configuration file, in seconds\n#\n# export OOZIE_LOG4J_RELOAD=10\n\n# The port Oozie server runs\n#\nexport OOZIE_HTTP_PORT={{oozie_server_port}}\n\n# The admin port Oozie server runs\n#\nexport OOZIE_ADMIN_PORT={{oozie_server_admin_port}}\n\n# The host name Oozie server runs on\n#\n# export OOZIE_HTTP_HOSTNAME=`hostname -f`\n\n# The base URL for callback URLs to Oozie\n#\n# export OOZIE_BASE_URL=\"http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie\"\nexport JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64\n\n# At least 1 minute of retry time to account for server downtime during\n# upgrade/downgrade\nexport OOZIE_CLIENT_OPTS=\"${OOZIE_CLIENT_OPTS} -Doozie.connection.retry.count=5 \"\n\n# This is needed so that Oozie does not run into OOM or GC Overhead limit\n# exceeded exceptions. If the oozie server is handling large number of\n# workflows/coordinator jobs, the memory setting
s may need to be revised\nexport CATALINA_OPTS=\"${CATALINA_OPTS} -Xmx2048m -XX:MaxPermSize=256m \""
+ },
+ "oozie-site": {
+ "oozie.authentication.simple.anonymous.allowed": "true",
+ "oozie.service.HadoopAccessorService.kerberos.enabled": "false",
+ "oozie.service.SchemaService.wf.ext.schemas": "shell-action-0.1.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,email-action-0.1.xsd,email-action-0.2.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,ssh-action-0.1.xsd,ssh-action-0.2.xsd,distcp-action-0.1.xsd,distcp-action-0.2.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd",
+ "oozie.service.coord.check.maximum.frequency": "false",
+ "oozie.services": "\n org.apache.oozie.service.SchedulerService,\n org.apache.oozie.service.InstrumentationService,\n org.apache.oozie.service.MemoryLocksService,\n org.apache.oozie.service.UUIDService,\n org.apache.oozie.service.ELService,\n org.apache.oozie.service.AuthorizationService,\n org.apache.oozie.service.UserGroupInformationService,\n org.apache.oozie.service.HadoopAccessorService,\n org.apache.oozie.service.JobsConcurrencyService,\n org.apache.oozie.service.URIHandlerService,\n org.apache.oozie.service.DagXLogInfoService,\n org.apache.oozie.service.SchemaService,\n org.apache.oozie.service.LiteWorkflowAppService,\n org.apache.oozie.service.JPAService,\n org.apache.oozie.service.StoreService,\n org.apache.oozie.service.CoordinatorStoreService,\n org.apache.oozie.service.SLAStoreService,\n org.apache.oozie.service.DBLiteWorkflowStoreService,\n org.apache.oozie.service.C
allbackService,\n org.apache.oozie.service.ShareLibService,\n org.apache.oozie.service.CallableQueueService,\n org.apache.oozie.service.ActionService,\n org.apache.oozie.service.ActionCheckerService,\n org.apache.oozie.service.RecoveryService,\n org.apache.oozie.service.PurgeService,\n org.apache.oozie.service.CoordinatorEngineService,\n org.apache.oozie.service.BundleEngineService,\n org.apache.oozie.service.DagEngineService,\n org.apache.oozie.service.CoordMaterializeTriggerService,\n org.apache.oozie.service.StatusTransitService,\n org.apache.oozie.service.PauseTransitService,\n org.apache.oozie.service.GroupsService,\n org.apache.oozie.service.ProxyUserService,\n org.apache.oozie.service.XLogStreamingService,\n org.apache.oozie.service.JvmPauseMonitorService"
+ },
+ "storm-env": {
+ "content": "\n#!/bin/bash\n\n# Set Storm specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\n\n# export STORM_CONF_DIR=\"\"\nSTORM_HOME=/usr/hdp/current/storm-client"
+ },
+ "storm-site": {
+ "_storm.min.ruid": "null",
+ "_storm.thrift.nonsecure.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
+ "_storm.thrift.secure.transport": "backtype.storm.security.auth.kerberos.KerberosSaslTransportPlugin",
+ "drpc.childopts": "-Xmx768m _JAAS_PLACEHOLDER",
+ "java.library.path": "/usr/local/lib:/opt/local/lib:/usr/lib:/usr/hdp/current/storm-client/lib",
+ "logviewer.childopts": "-Xmx128m _JAAS_PLACEHOLDER",
+ "nimbus.childopts": "-Xmx1024m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-nimbus/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "supervisor.childopts": "-Xmx256m _JAAS_PLACEHOLDER -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-supervisor/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "worker.childopts": "-Xmx768m _JAAS_PLACEHOLDER -javaagent:/usr/hdp/current/storm-client/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/hdp/current/storm-client/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "ui.childopts": "-Xmx768m _JAAS_PLACEHOLDER",
+ "storm.thrift.transport": {"remove": "yes"}
+ },
+ "tez-site": {
+ "tez.am.container.idle.release-timeout-max.millis": "20000",
+ "tez.am.container.idle.release-timeout-min.millis": "10000",
+ "tez.am.container.reuse.non-local-fallback.enabled": "false",
+ "tez.am.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.am.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.am.max.app.attempts": "2",
+ "tez.am.maxtaskfailures.per.node": "10",
+ "tez.am.resource.memory.mb": "1364",
+ "tez.cluster.additional.classpath.prefix": "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
+ "tez.counters.max": "2000",
+ "tez.counters.max.groups": "1000",
+ "tez.generate.debug.artifacts": "false",
+ "tez.grouping.max-size": "1073741824",
+ "tez.grouping.min-size": "16777216",
+ "tez.grouping.split-waves": "1.7",
+ "tez.history.logging.service.class": "org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService",
+ "tez.lib.uris": "/hdp/apps/${hdp.version}/tez/tez.tar.gz",
+ "tez.runtime.compress": "true",
+ "tez.runtime.compress.codec": "org.apache.hadoop.io.compress.SnappyCodec",
+ "tez.runtime.io.sort.mb": "272",
+ "tez.runtime.unordered.output.buffer.size-mb": "51",
+ "tez.session.client.timeout.secs": "-1",
+ "tez.shuffle-vertex-manager.max-src-fraction": "0.4",
+ "tez.shuffle-vertex-manager.min-src-fraction": "0.2",
+ "tez.task.am.heartbeat.counter.interval-ms.max": "4000",
+ "tez.task.launch.cluster-default.cmd-opts": "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
+ "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
+ "tez.task.launch.env": "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64",
+ "tez.task.max-events-per-heartbeat": "500",
+ "tez.task.resource.memory.mb": "682",
+ "tez.am.container.session.delay-allocation-millis": {"remove": "yes"},
+ "tez.am.env": {"remove": "yes"},
+ "tez.am.grouping.max-size": {"remove": "yes"},
+ "tez.am.grouping.min-size": {"remove": "yes"},
+ "tez.am.grouping.split-waves": {"remove": "yes"},
+ "tez.am.java.opts": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.max-src-fraction": {"remove": "yes"},
+ "tez.am.shuffle-vertex-manager.min-src-fraction": {"remove": "yes"},
+ "tez.runtime.intermediate-input.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-input.is-compressed": {"remove": "yes"},
+ "tez.runtime.intermediate-output.compress.codec": {"remove": "yes"},
+ "tez.runtime.intermediate-output.should-compress": {"remove": "yes"},
+ "tez.yarn.ats.enabled": {"remove": "yes"}
+ },
+ "webhcat-site": {
+ "templeton.hadoop": "/usr/hdp/current/hadoop-client/bin/hadoop",
+ "templeton.hcat": "/usr/hdp/current/hive-client/bin/hcat",
+ "templeton.hive.archive": "hdfs:///hdp/apps/${hdp.version}/hive/hive.tar.gz",
+ "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=thrift://vitaha-1.c.pramod-thangali.internal:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true",
+ "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+ "templeton.libjars": "/usr/hdp/current/zookeeper-client/zookeeper.jar",
+ "templeton.pig.archive": "hdfs:///hdp/apps/${hdp.version}/pig/pig.tar.gz",
+ "templeton.sqoop.archive": "hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop.tar.gz",
+ "templeton.sqoop.home": "sqoop.tar.gz/sqoop",
+ "templeton.sqoop.path": "sqoop.tar.gz/sqoop/bin/sqoop",
+ "templeton.streaming.jar": "hdfs:///hdp/apps/${hdp.version}/mapreduce/hadoop-streaming.jar"
+ },
+ "yarn-site": {
+ "hadoop.registry.rm.enabled": "false",
+ "yarn.application.classpath": "$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*",
+ "yarn.client.nodemanager-connect.max-wait-ms": "60000",
+ "yarn.client.nodemanager-connect.retry-interval-ms": "10000",
+ "yarn.node-labels.fs-store.retry-policy-spec": "2000, 500",
+ "yarn.node-labels.fs-store.root-dir": "/system/yarn/node-labels",
+ "yarn.node-labels.manager-class": "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager",
+ "yarn.nodemanager.bind-host": "0.0.0.0",
+ "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "90",
+ "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb": "1000",
+ "yarn.nodemanager.linux-container-executor.cgroups.hierarchy": "hadoop-yarn",
+ "yarn.nodemanager.linux-container-executor.cgroups.mount": "false",
+ "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage": "false",
+ "yarn.nodemanager.linux-container-executor.resources-handler.class": "org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler",
+ "yarn.nodemanager.log-aggregation.debug-enabled": "false",
+ "yarn.nodemanager.log-aggregation.num-log-files-per-app": "30",
+ "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "-1",
+ "yarn.nodemanager.recovery.dir": "{{yarn_log_dir_prefix}}/nodemanager/recovery-state",
+ "yarn.nodemanager.recovery.enabled": "true",
+ "yarn.nodemanager.resource.cpu-vcores": "1",
+ "yarn.nodemanager.resource.percentage-physical-cpu-limit": "100",
+ "yarn.resourcemanager.bind-host": "0.0.0.0",
+ "yarn.resourcemanager.connect.max-wait.ms": "900000",
+ "yarn.resourcemanager.connect.retry-interval.ms": "30000",
+ "yarn.resourcemanager.fs.state-store.retry-policy-spec": "2000, 500",
+ "yarn.resourcemanager.fs.state-store.uri": " ",
+ "yarn.resourcemanager.ha.enabled": "false",
+ "yarn.resourcemanager.recovery.enabled": "true",
+ "yarn.resourcemanager.state-store.max-completed-applications": "${yarn.resourcemanager.max-completed-applications}",
+ "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
+ "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size": "10",
+ "yarn.resourcemanager.system-metrics-publisher.enabled": "true",
+ "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled": "false",
+ "yarn.resourcemanager.work-preserving-recovery.enabled": "true",
+ "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms": "10000",
+ "yarn.resourcemanager.zk-acl": "world:anyone:rwcda",
+ "yarn.resourcemanager.zk-num-retries": "1000",
+ "yarn.resourcemanager.zk-retry-interval-ms": "1000",
+ "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore",
+ "yarn.resourcemanager.zk-timeout-ms": "10000",
+ "yarn.timeline-service.bind-host": "0.0.0.0",
+ "yarn.timeline-service.client.max-retries": "30",
+ "yarn.timeline-service.client.retry-interval-ms": "1000",
+ "yarn.timeline-service.http-authentication.simple.anonymous.allowed": "true",
+ "yarn.timeline-service.http-authentication.type": "simple",
+ "yarn.timeline-service.leveldb-timeline-store.read-cache-size": "104857600",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size": "10000",
+ "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size": "10000"
+ },
+ "hiveserver2-site": {
+ "hive.security.authenticator.manager": "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator",
+ "hive.security.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
+ },
+ "ranger-hbase-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hbase",
+ "REPOSITORY_CONFIG_USERNAME": "hbase",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "ranger-hbase-plugin-enabled": "No"
+
+ },
+ "ranger-hdfs-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hadoop",
+ "REPOSITORY_CONFIG_USERNAME": "hadoop",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "hadoop.rpc.protection": "-",
+ "ranger-hdfs-plugin-enabled": "No"
+ },
+ "ranger-hive-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "hive",
+ "REPOSITORY_CONFIG_USERNAME": "hive",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "UPDATE_XAPOLICIES_ON_GRANT_REVOKE": "true",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver",
+ "ranger-hive-plugin-enabled": "No"
+ },
+ "ranger-storm-plugin-properties": {
+ "REPOSITORY_CONFIG_PASSWORD": "stormclient",
+ "REPOSITORY_CONFIG_USERNAME": "stormclient@EXAMPLE.COM",
+ "SSL_KEYSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-keystore.jks",
+ "SSL_KEYSTORE_PASSWORD": "myKeyFilePassword",
+ "SSL_TRUSTSTORE_FILE_PATH": "/etc/hadoop/conf/ranger-plugin-truststore.jks",
+ "SSL_TRUSTSTORE_PASSWORD": "changeit",
+ "XAAUDIT.DB.IS_ENABLED": "true",
+ "XAAUDIT.HDFS.DESTINATION_DIRECTORY": "hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%",
+ "XAAUDIT.HDFS.DESTINTATION_FILE": "%hostname%-audit.log",
+ "XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS": "900",
+ "XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS": "86400",
+ "XAAUDIT.HDFS.IS_ENABLED": "false",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit/archive",
+ "XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT": "10",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY": "__REPLACE__LOG_DIR/hadoop/%app-type%/audit",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FILE": "%time:yyyyMMdd-HHmm.ss%.log",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS": "60",
+ "XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS": "600",
+ "common.name.for.certificate": "-",
+ "ranger-storm-plugin-enabled": "No"
+ }
+ }
+ }
+ ]
+}
[50/55] [abbrv] ambari git commit: AMBARI-18214. Restify Log Search
endpoints (oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VGroupListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VGroupListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VGroupListBase.js
index a34aaa3..0c81cc3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VGroupListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VGroupListBase.js
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,18 +17,18 @@
* under the License.
*/
-
+
define(['require',
'collections/BaseCollection',
'utils/Globals',
'models/VGroup'
],function(require,BaseCollection,Globals,VGroup){
- 'use strict';
+ 'use strict';
var VGroupListBase = BaseCollection.extend(
/** @lends VGroupListBase.prototype */
{
- url: Globals.baseURL + 'dashboard/hosts',
+ url: Globals.baseURL + 'service/logs/hosts',
model : VGroup,
@@ -44,7 +44,7 @@ define(['require',
this._changes = { };
this.on('change', this._onChange);
},
-
+
_onChange : function(m){
this._changes[m.id] = m;
},
@@ -59,7 +59,7 @@ define(['require',
getUsersOfGroup : function(groupId, options){
var url = Globals.baseURL + 'xusers/' + groupId + '/users';
-
+
options = _.extend({
//data : JSON.stringify(postData),
contentType : 'application/json',
@@ -80,5 +80,3 @@ define(['require',
return VGroupListBase;
});
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogLevelListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogLevelListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogLevelListBase.js
index 59b5ae8..0e3de20 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogLevelListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogLevelListBase.js
@@ -28,7 +28,7 @@ define(['require',
var VLogLevelListBase = BaseCollection.extend(
/** @lends VLogLevelListBase.prototype */
{
- url: Globals.baseURL + 'dashboard/levels_count',
+ url: Globals.baseURL + 'service/logs/levels/count',
model : VLogLevel,
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogListBase.js
index 7b102d5..72ab530 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VLogListBase.js
@@ -58,7 +58,7 @@ define(['require',
*************************/
cancelFindRequest : function(token, options){
- var url = Globals.baseURL + 'dashboard/cancelFindRequest';
+ var url = Globals.baseURL + 'service/logs/request/cancel';
options = _.extend({
data : $.param(token),
@@ -69,7 +69,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'GET', options);
},
getServiceLogFields : function(token, options){
- var url = Globals.baseURL + 'dashboard/getServiceLogsFieldsName';
+ var url = Globals.baseURL + 'service/logs/fields';
options = _.extend({
data : $.param(token),
@@ -80,7 +80,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'GET', options);
},
getServiceLogSchemaFields : function(token, options){
- var url = Globals.baseURL + 'dashboard/getServiceLogsSchemaFieldsName';
+ var url = Globals.baseURL + 'service/logs/schema/fields';
options = _.extend({
data : $.param(token),
@@ -91,7 +91,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'GET', options);
},
getTruncatedLogs : function(token, options){
- var url = Globals.baseURL + 'dashboard/getAfterBeforeLogs';
+ var url = Globals.baseURL + 'service/logs/truncated';
options = _.extend({
data : $.param(token),
@@ -102,7 +102,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'GET', options);
},
getServicesInfo : function(options){
- var url = Globals.baseURL + 'dashboard/getHadoopServiceConfigJSON';
+ var url = Globals.baseURL + 'service/logs/serviceconfig';
options = _.extend({
//data : $.param(token),
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNameValueListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNameValueListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNameValueListBase.js
index d59eaa2..71e80d9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNameValueListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNameValueListBase.js
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -17,18 +17,18 @@
* under the License.
*/
-
+
define(['require',
'collections/BaseCollection',
'utils/Globals',
'models/VNameValue'
],function(require,BaseCollection,Globals,VNameValue){
- 'use strict';
+ 'use strict';
var VNameValueListBase = BaseCollection.extend(
/** @lends VNameValueListBase.prototype */
{
- url: Globals.baseURL + 'dashboard/hosts',
+ url: Globals.baseURL + 'service/logs/hosts',
model : VNameValue,
@@ -42,7 +42,7 @@ define(['require',
this.modelAttrName = 'vnameValues';
this.bindErrorEvents();
}
-
+
},{
/**
* Table Cols to be passed to Backgrid
@@ -55,5 +55,3 @@ define(['require',
return VNameValueListBase;
});
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNodeListBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNodeListBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNodeListBase.js
index 7c7dcf8..78949e3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNodeListBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collection_bases/VNodeListBase.js
@@ -28,7 +28,7 @@ define(['require',
var VNodeListBase = BaseCollection.extend(
/** @lends VNodeListBase.prototype */
{
- url: Globals.baseURL + 'dashboard/getTreeExtension',
+ url: Globals.baseURL + 'service/logs/tree',
model : VNode,
@@ -48,7 +48,7 @@ define(['require',
*************************/
cancelFindRequest : function(token, options){
- var url = Globals.baseURL + 'dashboard/cancelFindRequest';
+ var url = Globals.baseURL + 'service/logs/request/cancel';
options = _.extend({
data : $.param(token),
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VAuditLogBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VAuditLogBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VAuditLogBase.js
index 1283875..8513ed2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VAuditLogBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VAuditLogBase.js
@@ -42,7 +42,7 @@ define(['require',
this.modelName = 'VAuditLogBase';
},
auditLiveFeed : function(token, options){
- var url = Globals.baseURL + 'audit/getLiveLogsCount';
+ var url = Globals.baseURL + 'audit/logs/live/count';
options = _.extend({
data : $.param(token),
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VCommonModelBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VCommonModelBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VCommonModelBase.js
index 4723e3e..bbc3e36 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VCommonModelBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VCommonModelBase.js
@@ -27,7 +27,7 @@ define(['require',
var VCommonModelBase = BaseModel.extend(
/** @lends VCommonModelBase.prototype */
{
- urlRoot: Globals.baseURL + 'dashboard/getTreeExtension',
+ urlRoot: Globals.baseURL + 'service/logs/tree',
defaults: {},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VEventHistoryBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VEventHistoryBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VEventHistoryBase.js
index c237ade..a15cdbd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VEventHistoryBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VEventHistoryBase.js
@@ -27,7 +27,7 @@ define(['require',
var VEventHistory = BaseModel.extend(
/** @lends VEventHistory.prototype */
{
- urlRoot: Globals.baseURL + 'dashboard/aggregatedData',
+ urlRoot: Globals.baseURL + 'service/logs/aggregated',
defaults: {},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VGraphInfoBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VGraphInfoBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VGraphInfoBase.js
index a707629..46e826c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VGraphInfoBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VGraphInfoBase.js
@@ -27,7 +27,7 @@ define(['require',
var VLogBase = BaseModel.extend(
/** @lends VLogBase.prototype */
{
- urlRoot: Globals.baseURL + 'dashboard/aggregatedData',
+ urlRoot: Globals.baseURL + 'service/logs/aggregated',
defaults: {},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VLogLevelBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VLogLevelBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VLogLevelBase.js
index 0384bc2..daaaa32 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VLogLevelBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VLogLevelBase.js
@@ -27,7 +27,7 @@ define(['require',
var VLogLevelBase = BaseModel.extend(
/** @lends VLogLevelBase.prototype */
{
- urlRoot: Globals.baseURL + 'dashboard/levels_count',
+ urlRoot: Globals.baseURL + 'service/logs/levels/count',
defaults: {},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VUserFilterBase.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VUserFilterBase.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VUserFilterBase.js
index 35171aa..da299a0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VUserFilterBase.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/model_bases/VUserFilterBase.js
@@ -27,7 +27,7 @@ define(['require',
var VUserFilterBase = BaseModel.extend(
/** @lends VUserFilterBase.prototype */
{
- urlRoot: Globals.baseURL + 'userconfig/user_filter',
+ urlRoot: Globals.baseURL + 'userconfig/users/filter',
defaults: {},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/ViewUtils.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/ViewUtils.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/ViewUtils.js
index 331ffd6..62d785b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/ViewUtils.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/ViewUtils.js
@@ -58,7 +58,7 @@ define(['require',
if (params.bundle_id && !params.start_time && !params.end_time) {
var collection = new VNameValueList();
- collection.url = Globals.baseURL + "dashboard/solr/getBundleIdBoundaryDates";
+ collection.url = Globals.baseURL + "service/logs/solr/boundarydates";
collection.modelAttrName = "vNameValues";
_.extend(collection.queryParams, {
"bundle_id": params.bundle_id
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
index c04aaf9..ef6dce5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditAggregatedView.js
@@ -98,7 +98,7 @@ define(['require',
pageSize: 9999
}
});
- this.topUsers.url = Globals.baseURL + "audit/getTopAuditUsers";
+ this.topUsers.url = Globals.baseURL + "audit/logs/users";
this.topUsers.modelAttrName = "graphData";
this.topResources = new VNameValueList([],{
state: {
@@ -106,7 +106,7 @@ define(['require',
pageSize: 9999
}
});
- this.topResources.url = Globals.baseURL + "audit/getTopAuditResources";
+ this.topResources.url = Globals.baseURL + "audit/logs/resources";
this.topResources.modelAttrName = "graphData";
//initialize colors
this.colors = (new d3.scale.category20c().range().slice().reverse()).concat(new d3.scale.category20b().range().slice().reverse());
@@ -408,7 +408,7 @@ define(['require',
obj.utcOffset = moment().utcOffset();
obj.startIndex = this.topUsers.state.currentPage * this.topUsers.state.pageSize;
var params = $.param(_.extend({},this.topUsers.queryParams,obj));
- var url = "api/v1/audit/exportUserTableToTextFile?"+ params;
+ var url = "api/v1/audit/logs/users/export?"+ params;
window.open(url);
this.onDialogClosed();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js
index 0b570ac..a399cbf 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js
@@ -27,9 +27,9 @@ define(['require',
'hbs!tmpl/audit/AuditTabLayoutView_tmpl',
'moment'
],function(require,Backbone,Globals,Utils,ViewUtils,VGroupList,VAuditLogList,VAuditLog,AuditTabLayoutViewTmpl,moment){
-
+
'use strict';
-
+
return Backbone.Marionette.Layout.extend(
/** @lends LogLevelView */
{
@@ -91,14 +91,14 @@ define(['require',
pageSize: 25
}
});
-
+
this.componentsList = new VGroupList([],{
state: {
firstPage: 0,
pageSize: 99999
}
});
- this.componentsList.url = Globals.baseURL + "audit/getAuditComponents";
+ this.componentsList.url = Globals.baseURL + "audit/logs/components";
},
bindEvents : function(){
this.listenTo(this.componentsList, "reset", function(col, abc){
@@ -115,13 +115,13 @@ define(['require',
this.listenTo(this.vent,"reinitialize:filter:mustNot",function(value){
this.reinitializeFilterMustNot(value);
},this);
-
+
this.listenTo(this.globalVent,"reinitialize:auditLogs",function(options){
this.vent.trigger("reinitialize:filter:mustNot reinitialize:filter:mustBe reinitialize:filter:logtime "+
"reinitialize:TopTenGraph",options);
this.fetchAuditLogs(options);
},this);
-
+
},
onRender : function(){
this.renderHistogram();
@@ -154,7 +154,7 @@ define(['require',
params : that.defaultParams
}));
})
-
+
},
fetchAuditColumns : function(){
var that =this;
@@ -184,7 +184,7 @@ define(['require',
renderVSSearch : function(){
var that = this;
require(['views/tabs/VisualSearchView'], function(VisualSearchView){
-
+
_.each(that.columns,function(v,i){
if(v.toLowerCase().indexOf("time") > 0 ){
//that.columns.splice(i, 1);
@@ -421,7 +421,7 @@ define(['require',
that.$("#loaderFeed").show();
},
success : function(data){
- var dd=[];
+ var dd=[];
that.$("#spark").parent().show();
_.each(data.vnameValues,function(d){
dd.push(d.value);
@@ -452,8 +452,8 @@ define(['require',
if(values.mustNot){
this.ui.excludeComponents.select2('val',values.mustNot.split(","));
}else{
- this.ui.excludeComponents.select2('val',[]);
+ this.ui.excludeComponents.select2('val',[]);
}
}
});
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/BubbleGraphTableLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/BubbleGraphTableLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/BubbleGraphTableLayoutView.js
index 42b94d5..4fb9b51 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/BubbleGraphTableLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/BubbleGraphTableLayoutView.js
@@ -31,7 +31,7 @@ define(['require',
'select2'
],function(require,Backbone,moment,Globals,Utils,ViewUtils,tip,VLogLevel,VLogList,VGraphInfo,BubbleGraphTableLayoutViewTmpl,JBDialog){
'use strict';
-
+
return Backbone.Marionette.Layout.extend(
/** @lends BubbleGraphTableLayoutView */
{
@@ -96,7 +96,7 @@ define(['require',
pageSize: 25
}
});
- this.collection.url = Globals.baseURL + "dashboard/solr/logs_search";
+ this.collection.url = Globals.baseURL + "service/logs";
this.graphModel = new VGraphInfo();
this.bindEvents();
this.graphParams = this.params;
@@ -257,7 +257,7 @@ define(['require',
filterOpts : {},
paginatorOpts : {}
}));
- });
+ });
},
renderComponentList : function(){
var that = this;
@@ -416,7 +416,7 @@ define(['require',
displayOrder :6,
width : 6
}
-
+
};
_.each(cols,function(c,k){
if(columns[k] == undefined){
@@ -502,7 +502,7 @@ define(['require',
this.ui.hostList.show();
}else
this.ui.componentList.show();
-
+
},
bindContextMenuClick : function(){
var that = this;
@@ -543,13 +543,13 @@ define(['require',
that.selectionCallBack(selection,e)
},1);
-
+
});
},
selectionCallBack : function(selection,e){
this.RLogTable.currentView.$el.removeHighlight(true);
if(this.selectionText != selection.toString()){
- this.selectionText = selection.toString();
+ this.selectionText = selection.toString();
}else{
$(".contextMenuBody [data-id='F']").show();
$(".contextMenuBody").hide();
@@ -561,8 +561,8 @@ define(['require',
$(".contextMenuBody").show();
$(".contextMenuBody").css({
'top':e.pageY - 40,
- 'left':e.pageX
- });
+ 'left':e.pageX
+ });
}else{
this.RLogTable.currentView.$el.removeHighlight(true);
$(".contextMenuBody [data-id='F']").show();
@@ -607,7 +607,7 @@ define(['require',
}else if ($el.data("id") === "C_M"){
this.globalVent.trigger("add:compare",$el.find('a'));
}
-
+
},
renderDetailLogFileView : function(view){
var that = this;
@@ -655,12 +655,12 @@ define(['require',
this.ui.graph.text("no data");
return
}
-
+
var root = {
name : "",
dataList : this.graphModel.get("graphData")
};
-
+
var margin = 20;
this.ui.graph.empty();
// var color = d3.scale.linear()
@@ -813,6 +813,6 @@ define(['require',
$('body').unbind("mouseup.contextMenu");
}
});
-
-
-});
\ No newline at end of file
+
+
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentListView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentListView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentListView.js
index abd3740..b8c43bb 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentListView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentListView.js
@@ -68,14 +68,14 @@ define(['require',
pageSize: 99999
}
});
- this.componentsList.url = Globals.baseURL + "dashboard/getComponentListWithLevelCounts";
+ this.componentsList.url = Globals.baseURL + "service/logs/components/level/counts";
this.hostList = new VNodeList([],{
state: {
firstPage: 0,
pageSize: 99999
}
});
- this.hostList.url = Globals.baseURL + "dashboard/getHostListByComponent";
+ this.hostList.url = Globals.baseURL + "service/logs/hosts/components";
},
/** all events binding here */
bindEvents : function(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentsView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentsView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentsView.js
index 66cc277..424236d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentsView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/ComponentsView.js
@@ -53,7 +53,7 @@ define(['require',
initialize: function(options) {
_.extend(this, _.pick(options,'vent'));
this.collection = new VLogLevel();
- this.collection.url = Globals.baseURL+"dashboard/components_count";
+ this.collection.url = Globals.baseURL+"service/logs/components/count";
this.bindEvents();
},
onRender : function(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/DashboardView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/DashboardView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/DashboardView.js
index c3fd9c2..35814ba 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/DashboardView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/DashboardView.js
@@ -26,7 +26,7 @@ define(['require',
],function(require,Backbone,Globals,DashboardviewTmpl,VGroupList){
'use strict';
-
+
var DashboardView = Backbone.Marionette.Layout.extend(
/** @lends DashboardView */
{
@@ -69,7 +69,7 @@ define(['require',
this.logRegions = new Backbone.Collection();
this.hostCollection = new VGroupList([],{});
this.cComponents = new VGroupList([],{});
- this.cComponents.url = Globals.baseURL + "dashboard/components";
+ this.cComponents.url = Globals.baseURL + "service/logs/components";
this.cTime = new VGroupList(Globals.timeQueryLOV,{});
},
/** all events binding here */
@@ -115,7 +115,7 @@ define(['require',
},
onSearchLogClick : function(e){
var searchParams = this.getSearchparams();
-
+
if(this.logRegions.length == 0){
var model = new Backbone.Model({
id : 1,
@@ -123,7 +123,7 @@ define(['require',
});
this.generateView(model);
}else{
-
+
var existsMod = this.logRegions.find(function(m){
return JSON.stringify(searchParams) === JSON.stringify(m.get('params'))
});
@@ -159,7 +159,7 @@ define(['require',
require(['views/dashboard/LogDetailView'],function(LogDetailView){
region.show(new LogDetailView({model: model}));
})
-
+
},
getSearchparams : function(){
var obj={hosts : null, components : null, time:null};
@@ -176,7 +176,7 @@ define(['require',
obj.time = this.ui.time.select2("val");
}
return obj;
-
+
},
/** on close */
onClose: function(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/GridTableLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/GridTableLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/GridTableLayoutView.js
index 1cbdef8..da930ea 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/GridTableLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/GridTableLayoutView.js
@@ -78,7 +78,7 @@ define(['require',
this.graphParams = {};
this.unit = (this.model.params && this.model.params.unit) ? this.model.params.unit : "+1HOUR";
this.firstRender = true;
- this.collection.url = Globals.baseURL + "audit/getAnyGraphData";
+ this.collection.url = Globals.baseURL + "audit/logs/anygraph";
this.collection.modelAttrName = "graphData";
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/HostsView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/HostsView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/HostsView.js
index dd82130..2440fd0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/HostsView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/HostsView.js
@@ -53,7 +53,7 @@ define(['require',
initialize: function(options) {
_.extend(this, _.pick(options,'vent'));
this.collection = new VLogLevel();
- this.collection.url = Globals.baseURL+"dashboard/hosts_count";
+ this.collection.url = Globals.baseURL+"service/logs/hosts/count";
this.bindEvents();
},
onRender : function(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
index b57f7c1..d2ee3d8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js
@@ -61,7 +61,7 @@ define(['require',
initialize: function(options) {
_.extend(this, _.pick(options,'vent','globalVent','params'));
this.logLevelList = new VLogLevelList();
- this.logLevelList.url = Globals.baseURL + "dashboard/getLogLevelCounts";
+ this.logLevelList.url = Globals.baseURL + "service/logs/levels/counts/namevalues";
this.logLevelList.modelAttrName = "vNameValues";
this.bindEvents();
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js
index 3be87f5..66593c4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/MainLayoutView.js
@@ -27,7 +27,7 @@ define(['require',
'd3.tip'
],function(require,Backbone,Utils,ViewUtils,Globals,MainLayoutViewTmpl){
'use strict';
-
+
var MainLayoutView = Backbone.Marionette.Layout.extend(
/** @lends MainLayoutView */
{
@@ -93,7 +93,7 @@ define(['require',
// pageSize: 50
// }
// });
-// this.collection.url = Globals.baseURL + "dashboard/solr/logs_search";
+// this.collection.url = Globals.baseURL + "service/logs";
this.vent = new Backbone.Wreqr.EventAggregator();
this.dateUtil = Utils.dateUtil;
this.bindEvents();
@@ -107,7 +107,7 @@ define(['require',
setTimeout(function(){
that.reAdjustTab()
},1000);
-
+
},this);
this.listenTo(this.globalVent,"render:comparison:tab",function(options){
this.hideContextMenu();
@@ -211,7 +211,7 @@ define(['require',
$("html, body").animate({ scrollTop: 0 }, 500);
that.showTab(tabName);
});
-
+
},
showTab : function(tabId){
this.$(".nav.nav-tabs li").removeClass("active");
@@ -242,7 +242,7 @@ define(['require',
globalVent:that.globalVent
}));
})
-
+
},
renderComponents : function(){
var that = this;
@@ -274,7 +274,7 @@ define(['require',
renderTroubleShootTab:function(){
var that = this;
require(['views/troubleshoot/TroubleShootLayoutView'], function(TroubleShootLayoutView){
-
+
that.RTroubleShoot.show(new TroubleShootLayoutView({
globalVent:that.globalVent
}));
@@ -345,7 +345,7 @@ define(['require',
var pack = d3.layout.pack()
.padding(2)
.size([diameter - margin, diameter - margin])
- .value(function(d) {
+ .value(function(d) {
return d.count; })
.children(function(d){
return d.dataList;
@@ -381,9 +381,9 @@ define(['require',
var circle = svg.selectAll("circle")
.data(nodes)
.enter().append("circle")
- .attr("class", function(d) {
+ .attr("class", function(d) {
return d.parent ? d.children ? "node" : "node node--leaf "+d.name : "node node--root"; })
- .style("fill", function(d) {
+ .style("fill", function(d) {
return d.children ? color(d.depth) : null; })
.on("click", function(d) {
if(d3.event.shiftKey){
@@ -391,7 +391,7 @@ define(['require',
}else{
if (focus !== d) zoom(d), d3.event.stopPropagation();
}
-
+
})
.on('mouseover', function (d,i) {
if (d.x) {
@@ -403,7 +403,7 @@ define(['require',
tipCirclePack.hide(d);
}
});
-
+
var text = svg.selectAll("text")
.data(nodes)
.enter().append("text")
@@ -418,7 +418,7 @@ define(['require',
return "";
}else
return d.name;
-
+
});
var node = svg.selectAll("circle,text");
@@ -472,7 +472,7 @@ define(['require',
el.find('i').removeClass('fa-square-o').addClass('fa-check-square-o');
this.quickMenuCompare = false;
this.onCompareLink(el);
- }
+ }
}else{
el.find('i').removeClass('fa-square-o').addClass('fa-check-square-o');
}
@@ -643,7 +643,7 @@ define(['require',
else {
that.$('.scroller-right').hide();
}
-
+
if (getLeftPosi()<0) {
that.$('.scroller-left').show();
}
@@ -655,7 +655,7 @@ define(['require',
this.$('.scroller-right').click(function(e) {
-
+
/* that.$('.scroller-left').fadeIn('slow');
that.$('.scroller-right').fadeOut('slow');*/
//console.log(widthOfHidden())
@@ -664,23 +664,23 @@ define(['require',
that.reAdjustTab();
});
}
-
+
});
this.$('.scroller-left').click(function() {
-
+
/*that.$('.scroller-right').fadeIn('slow');
that.$('.scroller-left').fadeOut('slow');*/
//console.log(getLeftPosi())
if(getLeftPosi() < 0){
that.$('.list').animate({left:"-="+(-40)+"px"},0,function(){
that.reAdjustTab();
- });
+ });
}
-
- });
+
+ });
},
});
return MainLayoutView;
-
-});
\ No newline at end of file
+
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/GlobalExclusionCompositeView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/GlobalExclusionCompositeView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/GlobalExclusionCompositeView.js
index a737eba..ffef322 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/GlobalExclusionCompositeView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/GlobalExclusionCompositeView.js
@@ -50,7 +50,7 @@ define(['require',
pageSize: 1000
}
});
- this.componentsList.url = Globals.baseURL + "dashboard/components";
+ this.componentsList.url = Globals.baseURL + "service/logs/components";
this.collection = this.exclusionObj.logMessageCollection;
if (this.collection.length == 0) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
index ba07600..95324d9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js
@@ -52,7 +52,7 @@ define(['require',
events["click [data-value]"] = 'onLogLevelHeaderClick';
events["click #filterContent input[type='checkbox']"] = 'onAnyCheckboxClick';
events["click .overrideRow a"] = 'onEditHost';
-
+
return events;
},
@@ -76,8 +76,8 @@ define(['require',
}
});
- this.componentsList.url = Globals.baseURL + "dashboard/components";
- this.hostList.url = Globals.baseURL + "dashboard/hosts";
+ this.componentsList.url = Globals.baseURL + "service/logs/components";
+ this.hostList.url = Globals.baseURL + "service/logs/hosts";
this.model = new VUserFilter();
this.levelCollection = new Backbone.Collection();
@@ -100,7 +100,7 @@ define(['require',
onRender: function() {
var that = this;
// this.setupSelect2Fields(this.levelCollection, "type", "type", "levelSelect2", 'Select Level');
-
+
$.when(this.hostList.fetch({ reset: true }), this.componentsList.fetch({ reset: true }), this.model.fetch({})).then(function(c1, c2, m1) {
// if (!_.isUndefined(that.model.get('components'))) {
// that.ui.componentSelect2.select2('val', that.model.get('components'));
@@ -116,7 +116,7 @@ define(['require',
//that.dataLevels = [];
//that.dataLevels = _.pluck(that.levelCollection.models, 'attributes');
-
+
//that.dataList = [];
//that.dataList = _.pluck(that.componentsList.models, 'attributes');
that.renderComponents();
@@ -148,13 +148,13 @@ define(['require',
var components = this.model.get("filter");
_.each(components,function(value,key){
var obj = components[key];
-
- if((_.isArray(obj.overrideLevels) && obj.overrideLevels.length) ||
+
+ if((_.isArray(obj.overrideLevels) && obj.overrideLevels.length) ||
(_.isArray(obj.hosts) && obj.hosts.length) || obj.expiryTime){
var $el = that.$("input[data-name='"+key+"']").filter("[data-override]");
$el.click();
}
-
+
//setting override data
if(_.isArray(obj.overrideLevels)){
if(obj.overrideLevels.length){
@@ -202,7 +202,7 @@ define(['require',
_.each(this.levelCollection.models,function(model){
that.setCheckAllValue(model.get("type"));
});
-
+
},
onAnyCheckboxClick : function(e){
var $el = $(e.currentTarget);
@@ -363,4 +363,4 @@ define(['require',
}
});
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
index 0085f06..76d0537 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js
@@ -81,10 +81,10 @@ define(['require',
this.firstRender = true;
if (!this.viewType || this.viewType == Globals.graphType.HISTOGRAM.value) {
this.histogramView = true;
- this.collection.url = Globals.baseURL + "dashboard/getHistogramData";
+ this.collection.url = Globals.baseURL + "service/logs/histogram";
this.collection.modelAttrName = "graphData";
} else {
- this.collection.url = Globals.baseURL + "audit/getAuditLineGraphData";
+ this.collection.url = Globals.baseURL + "audit/logs/linegraph";
this.collection.modelAttrName = "graphData";
this.lineView = true;
}
@@ -246,7 +246,7 @@ define(['require',
}
}
}
- Obj.values = newObj;
+ Obj.values = newObj;
}
if (that.histogramView) {
@@ -282,7 +282,7 @@ define(['require',
generate: function() {
/* var parentWidth = (that.ui.histoGraph.find('svg').parent().width()),
parentHeight = (that.ui.histoGraph.find('svg').parent().height())
- width = ((parentWidth === 0) ? (891) : (parentWidth)), // -15 because parent has 15 padding
+ width = ((parentWidth === 0) ? (891) : (parentWidth)), // -15 because parent has 15 padding
height = ((parentHeight === 0) ? (640) : (parentHeight)) // -15 because parent has 15 padding */
if (that.histogramView) {
that.chart = nv.models.multiBarChart()
@@ -290,7 +290,7 @@ define(['require',
.height(height)*/
.stacked(true)
.showControls(false);
- that.chart.groupSpacing(0.6) // for bar width and aspace
+ that.chart.groupSpacing(0.6) // for bar width and aspace
} else {
that.chart = nv.models.lineChart().options({
transitionDuration: 300,
@@ -484,7 +484,7 @@ define(['require',
that.vent.trigger("date:setDate", {
'from': that.dateUtil.getMomentObject(that.brushValue[0]),
'to': that.dateUtil.getMomentObject(that.brushValue[1])
- });
+ });
}
/*}*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
index b0339f3..47393bb 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js
@@ -72,7 +72,7 @@ define(['require',
}
});
- this.collection.url = Globals.baseURL + "audit/getAnyGraphData";
+ this.collection.url = Globals.baseURL + "audit/logs/anygraph";
this.collection.modelAttrName = "graphData";
}
this.dateUtil = Utils.dateUtil;
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonView.js
index 1d26dc4..f922087 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/ComparisonView.js
@@ -65,7 +65,7 @@ define(['require',
}
});
this.dateUtil = Utils.dateUtil;
- this.collection.url = Globals.baseURL + "dashboard/solr/logs_search";
+ this.collection.url = Globals.baseURL + "service/logs";
this.bindEvents();
this.dateRangeLabel = "Last 1 Hour"
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js
index 6c6a77e..e4d327c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js
@@ -79,7 +79,7 @@ define(['require',
initialize: function(options) {
_.extend(this, _.pick(options,'globalVent'));
// this.logLevelList = new VLogLevelList();
-// this.logLevelList.url = Globals.baseURL + "dashboard/getLogLevelCounts";
+// this.logLevelList.url = Globals.baseURL + "service/logs/levels/counts/namevalues";
// this.logLevelList.modelAttrName = "vNameValues";
this.columnCollection = new VLogList([],{
state: {
@@ -93,15 +93,15 @@ define(['require',
pageSize: 99999
}
});
- this.componentsList.url = Globals.baseURL + "dashboard/components";
+ this.componentsList.url = Globals.baseURL + "service/logs/components";
this.vent = new Backbone.Wreqr.EventAggregator();
-
+
this.defaultParams = ViewUtils.getDefaultParamsForHierarchy();
this.bindEvents();
},
applyParamsDate:function(date){
if (date) {
- var dateString = date.split(',');
+ var dateString = date.split(',');
if(dateString.length){
var checkDate = Utils.dateUtil.getMomentUTC(dateString[0]);
if(checkDate.isValid()){
@@ -112,7 +112,7 @@ define(['require',
}
return checkDate.toJSON();
}
- }
+ }
}
},
bindEvents : function(){
@@ -144,7 +144,7 @@ define(['require',
this.listenTo(this.vent,"tab:refresh",function(params){
this.reRenderComponents(params);
},this);
-
+
this.listenTo(this.globalVent,"reinitialize:serviceLogs",function(options){
this.vent.trigger("reinitialize:filter:tree reinitialize:filter:include:exclude reinitialize:filter:bubbleTable"+
" reinitialize:filter:mustNot reinitialize:filter:mustBe reinitialize:filter:level reinitialize:filter:logtime",options);
@@ -163,7 +163,7 @@ define(['require',
this.componentsList.fetch({reset:true});
},
onShow:function(){
-
+
//this.REventHistory.currentView.genrateTimeline();
},
@@ -267,7 +267,7 @@ define(['require',
params : that.defaultParams,
}));
});
-
+
},
renderVSSearch : function(){
@@ -294,7 +294,7 @@ define(['require',
query : query
};
}
-
+
}));
});
},
@@ -309,7 +309,7 @@ define(['require',
placeholder : "Include Search",
vent : that.vent,
globalVent:that.globalVent,
- customOptions : columns,
+ customOptions : columns,
eventName : Globals.eventName.serviceLogsIncludeColumns,
myFormatData : function(query,searchCollection){
var obj=[];
@@ -431,7 +431,7 @@ define(['require',
this.ui.excludeComponents.select2('val',values.mustNot.split(","));
else
this.ui.excludeComponents.select2('val',[]);
-
+
},
reRenderComponents : function(params){
var iComponents = this.ui.includeComponents.val(),eComponents = this.ui.excludeComponents.val(),that=this;
@@ -455,20 +455,20 @@ define(['require',
} else{
this.ui.advanceSearch.hide();
this.ui.applySearch.hide();
- this.ui.basicSearch.show();
+ this.ui.basicSearch.show();
obj = this.getIncludeExcludeColValues();
obj.advanceSearch = null;
}
-
+
this.vent.trigger('main:search',obj);
-
+
},
applySearchBtn : function(){
var obj = {}
obj.advanceSearch = this.RAdvanceSearch.currentView.ui.searchArea.val();
this.vent.trigger('main:search',obj);
-
+
},
getIncludeExcludeColValues : function(){
return _.extend(this.RVisualSearchIncCol.currentView.formatData(this.RVisualSearchIncCol.currentView.visualSearch.searchBox.value(),this.RVisualSearchIncCol.currentView.visualSearch.searchQuery),
@@ -483,6 +483,6 @@ define(['require',
}
}
});
-
-
-});
\ No newline at end of file
+
+
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
index 4af4670..4b42eab 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js
@@ -99,7 +99,7 @@ define(['require',
pageSize: 25
}
});
- this.logFileCollection.url = Globals.baseURL + "dashboard/solr/logs_search";
+ this.logFileCollection.url = Globals.baseURL + "service/logs";
this.vent = new Backbone.Wreqr.EventAggregator();
this.bindEvents();
this.commonTableOptions = {
@@ -203,7 +203,7 @@ define(['require',
// {startIndex : this.logFileCollection.state.currentPage * this.logFileCollection.state.pageSize},obj),
// 'component','from','to','host','level','unit','startIndex','pageSize','format','utcOffset'));
var params = $.param(_.extend({}, this.logFileCollection.queryParams, obj));
- var url = "api/v1/dashboard/exportToTextFile?" + params;
+ var url = "api/v1/service/logs/export?" + params;
window.open(url);
this.onDialogClosed();
},
@@ -256,7 +256,7 @@ define(['require',
var top = element.offset().top;
element.addClass('highlightLog');
$("html, body").animate({ scrollTop: (top - 200) }, 1);
- /*setTimeout(function(){
+ /*setTimeout(function(){
element.addClass('fadeOutColor')
setTimeout(function(){element.removeClass('fadeOutColor highlightLog');},4000)
},6000);*/
@@ -599,7 +599,7 @@ define(['require',
},
initializeContextMenu: function() {
var that = this;
-
+
$('body').on("mouseup.contextMenuLogFile", function(e) {
var selection;
if (window.getSelection) {
@@ -657,7 +657,7 @@ define(['require',
}else if(type === "IA" || type === "EA"){
this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns",
key:Globals.serviceLogsColumns["log_message"],value:"*"+this.selectionText+"*"});
- }
+ }
else {
//this.vent.trigger("add:include:exclude",{type:type,value:this.selectionText});
this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns", key: Globals.serviceLogsColumns["log_message"], value: this.selectionText });
@@ -892,4 +892,4 @@ define(['require',
$('body').unbind("mouseup.contextMenuLogFile");
}
});
-});
\ No newline at end of file
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/TreeView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/TreeView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/TreeView.js
index cf33e68..bcd2c80 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/TreeView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/TreeView.js
@@ -80,7 +80,7 @@ define(['require',
initialize: function(options) {
_.extend(this, _.pick(options,'vent','globalVent','params'));
// this.collection = new VLogLevel();
-// this.collection.url = Globals.baseURL+"dashboard/components_count";
+// this.collection.url = Globals.baseURL+"service/logs/components/count";
this.treeModel = new VCommonModel();
this.searchParams = (this.params)? this.params :{};
this.bindEvents();
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
index c1655f0..66f851c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js
@@ -82,7 +82,7 @@ define(['backbone',
}
});
- this.serviceLogsCollection.url = Globals.baseURL + "dashboard/getAnyGraphData";
+ this.serviceLogsCollection.url = Globals.baseURL + "service/logs/anygraph";
this.serviceLogsCollection.modelAttrName = "graphData";
this.topUsers = new VNameValueList([],{
@@ -91,7 +91,7 @@ define(['backbone',
pageSize: 9999
}
});
- this.topUsers.url = Globals.baseURL + "audit/getTopAuditUsers";
+ this.topUsers.url = Globals.baseURL + "audit/logs/users";
this.topUsers.modelAttrName = "graphData";
this.serviceLoadCollection = new VLogList([], {
@@ -101,7 +101,7 @@ define(['backbone',
}
});
- this.serviceLoadCollection.url = Globals.baseURL + "audit/getServiceLoad";
+ this.serviceLoadCollection.url = Globals.baseURL + "audit/logs/serviceload";
this.serviceLoadCollection.modelAttrName = "graphData";
},
bindEvents : function(){
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/graphs/backup.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/graphs/backup.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/graphs/backup.js
index 9b589f1..8999e01 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/graphs/backup.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/graphs/backup.js
@@ -78,10 +78,10 @@ define(['require',
this.firstRender = true;
if (!this.viewType || this.viewType == Globals.graphType.HISTOGRAM.value) {
this.histogramView = true;
- this.collection.url = Globals.baseURL + "dashboard/getHistogramData";
+ this.collection.url = Globals.baseURL + "service/logs/histogram";
this.collection.modelAttrName = "graphData";
} else {
- this.collection.url = Globals.baseURL + "audit/getAuditLineGraphData";
+ this.collection.url = Globals.baseURL + "audit/logs/linegraph";
this.collection.modelAttrName = "graphData";
this.lineView = true;
}
@@ -323,7 +323,7 @@ define(['require',
generate: function() {
/* var parentWidth = (that.ui.histoGraph.find('svg').parent().width()),
parentHeight = (that.ui.histoGraph.find('svg').parent().height())
- width = ((parentWidth === 0) ? (891) : (parentWidth)), // -15 because parent has 15 padding
+ width = ((parentWidth === 0) ? (891) : (parentWidth)), // -15 because parent has 15 padding
height = ((parentHeight === 0) ? (640) : (parentHeight)) // -15 because parent has 15 padding */
if (that.histogramView) {
that.chart = nv.models.multiBarChart()
@@ -331,7 +331,7 @@ define(['require',
.height(height)*/
.stacked(true)
.showControls(false);
- that.chart.groupSpacing(0.6) // for bar width and aspace
+ that.chart.groupSpacing(0.6) // for bar width and aspace
} else {
that.chart = nv.models.lineChart().options({
transitionDuration: 300,
@@ -444,8 +444,8 @@ define(['require',
that.$(".box").resize(function() {
// var parentWidth = (that.ui.histoGraph.find('svg').parent().width()),
// parentHeight = (that.ui.histoGraph.find('svg').parent().height()),
- // width = ((parentWidth === 0) ? (891) : (parentWidth)), // -15 because parent has 15 padding
- // height = ((parentHeight === 0) ? (240) : (parentHeight)) // -15 because parent has 15 padding
+ // width = ((parentWidth === 0) ? (891) : (parentWidth)), // -15 because parent has 15 padding
+ // height = ((parentHeight === 0) ? (240) : (parentHeight)) // -15 because parent has 15 padding
// graph.width(width).height(height);
// if (that.parentView) {
// var elem = that.parentView.$el.find('[data-id="rHistogramGraph"] svg');
http://git-wip-us.apache.org/repos/asf/ambari/blob/0567b179/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
index 276a65e..061c607 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingRequestHelperImpl.java
@@ -62,9 +62,9 @@ public class LoggingRequestHelperImpl implements LoggingRequestHelper {
private static final String LOGSEARCH_ADMIN_PASSWORD_PROPERTY_NAME = "logsearch_admin_password";
- private static final String LOGSEARCH_QUERY_PATH = "/service/dashboard/solr/logs_search";
+ private static final String LOGSEARCH_QUERY_PATH = "/api/v1/service/logs";
- private static final String LOGSEARCH_GET_LOG_LEVELS_PATH = "/service/dashboard/getLogLevelCounts";
+ private static final String LOGSEARCH_GET_LOG_LEVELS_PATH = "/api/v1/service/logs/levels/counts/namevalues";
private static final String LOGSEARCH_ADMIN_CREDENTIAL_NAME = "logsearch.admin.credential";