You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by am...@apache.org on 2018/01/10 11:00:13 UTC
[ambari] 12/15: AMBARI-22674. ServiceAdvisor for OneFS (amagyar)
This is an automated email from the ASF dual-hosted git repository.
amagyar pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 89f3bba6b100288f736a73096ea3578153b20b1c
Author: Attila Magyar <am...@hortonworks.com>
AuthorDate: Fri Dec 22 09:54:10 2017 +0100
AMBARI-22674. ServiceAdvisor for OneFS (amagyar)
---
.../main/python/ambari_agent/alerts/ams_alert.py | 4 +-
.../main/python/ambari_agent/alerts/base_alert.py | 47 +-------
.../python/ambari_agent/alerts/metric_alert.py | 3 +-
.../main/python/ambari_agent/alerts/port_alert.py | 4 +-
.../main/python/ambari_agent/alerts/web_alert.py | 6 +-
.../src/main/python/ambari_commons/inet_utils.py | 46 +++++++
.../management-packs/isilon-onefs-mpack/pom.xml | 23 ++++
.../ONEFS/1.0.0/configuration/core-site.xml | 14 +--
.../ONEFS/1.0.0/configuration/hdfs-site.xml | 24 ++--
.../ONEFS/1.0.0/configuration/onefs.xml | 30 +++++
.../addon-services/ONEFS/1.0.0/metainfo.xml | 13 ++
.../ONEFS/1.0.0/package/scripts/params_linux.py | 12 +-
.../addon-services/ONEFS/1.0.0/service_advisor.py | 122 +++++++++++++++++++
.../addon-services/ONEFS/1.0.0/themes/theme.json | 60 ++++++++++
.../src/test/python/TestServiceAdvisor.py | 67 +++++++++++
.../src/test/python/unitTests.py | 132 +++++++++++++++++++++
16 files changed, 529 insertions(+), 78 deletions(-)
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
index 29b885b..cc2884e 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/ams_alert.py
@@ -22,7 +22,6 @@ import httplib
import imp
import time
import urllib
-from alerts.base_alert import BaseAlert
from alerts.metric_alert import MetricAlert
import ambari_simplejson as json
import logging
@@ -30,6 +29,7 @@ import re
import uuid
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
+from ambari_commons import inet_utils
logger = logging.getLogger()
@@ -68,7 +68,7 @@ class AmsAlert(MetricAlert):
logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
- host = BaseAlert.get_host_from_url(alert_uri.uri)
+ host = inet_utils.get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
index 5c0305e..cdbe316 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py
@@ -510,49 +510,4 @@ class BaseAlert(object):
:param state: the state of the alert in uppercase (such as OK, WARNING, etc)
:return: the parameterized text
'''
- return '{0}'
-
- """
- See RFC3986, Appendix B
- Tested on the following cases:
- "192.168.54.1"
- "192.168.54.2:7661
- "hdfs://192.168.54.3/foo/bar"
- "ftp://192.168.54.4:7842/foo/bar"
-
- Returns None if only a port is passed in
- """
- @staticmethod
- def get_host_from_url(uri):
- if uri is None:
- return None
-
- # if not a string, return None
- if not isinstance(uri, basestring):
- return None
-
- # RFC3986, Appendix B
- parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
-
- # index of parts
- # scheme = 1
- # authority = 3
- # path = 4
- # query = 6
- # fragment = 8
-
- host_and_port = uri
- if 0 == len(parts[0][1]):
- host_and_port = parts[0][4]
- elif 0 == len(parts[0][2]):
- host_and_port = parts[0][1]
- elif parts[0][2].startswith("//"):
- host_and_port = parts[0][3]
-
- if -1 == host_and_port.find(':'):
- if host_and_port.isdigit():
- return None
-
- return host_and_port
- else:
- return host_and_port.split(':')[0]
+ return '{0}'
\ No newline at end of file
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
index 66a1d05..64a1a29 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py
@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
from ambari_commons.urllib_handlers import RefreshHeaderProcessor
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_commons import inet_utils
from ambari_agent import Constants
logger = logging.getLogger()
@@ -80,7 +81,7 @@ class MetricAlert(BaseAlert):
logger.debug("[Alert][{0}] Calculated metric URI to be {1} (ssl={2})".format(
self.get_name(), alert_uri.uri, str(alert_uri.is_ssl_enabled)))
- host = BaseAlert.get_host_from_url(alert_uri.uri)
+ host = inet_utils.get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
index 02cc91c..1e2e581 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/port_alert.py
@@ -24,7 +24,7 @@ import time
from alerts.base_alert import BaseAlert
from resource_management.libraries.functions.get_port_from_url import get_port_from_url
from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address
+from ambari_commons.inet_utils import resolve_address, get_host_from_url
logger = logging.getLogger()
# default timeouts
@@ -111,7 +111,7 @@ class PortAlert(BaseAlert):
break
- host = BaseAlert.get_host_from_url(uri_value)
+ host = get_host_from_url(uri_value)
if host is None or host == "localhost" or host == "0.0.0.0":
host = self.host_name
host_not_specified = True
diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
index 0e400f7..6e01b89 100644
--- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
+++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py
@@ -21,9 +21,7 @@ limitations under the License.
import logging
import time
import urllib2
-import ssl
-from functools import wraps
from urllib2 import HTTPError
from tempfile import gettempdir
@@ -33,7 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
from resource_management.libraries.functions.get_path_from_url import get_path_from_url
from resource_management.libraries.functions.curl_krb_request import curl_krb_request
from ambari_commons import OSCheck
-from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol
+from ambari_commons.inet_utils import resolve_address, ensure_ssl_using_protocol, get_host_from_url
from ambari_agent import Constants
from ambari_agent.AmbariConfig import AmbariConfig
@@ -135,7 +133,7 @@ class WebAlert(BaseAlert):
uri_path = get_path_from_url(string_uri)
# start building the URL manually
- host = BaseAlert.get_host_from_url(alert_uri.uri)
+ host = get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 5fd3e88..a226235 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -23,6 +23,7 @@ import time
import sys
import urllib2
import socket
+import re
from ambari_commons import OSCheck
from functools import wraps
@@ -266,3 +267,48 @@ def ensure_ssl_using_protocol(protocol="PROTOCOL_TLSv1", ca_certs=None):
return context
_create_default_https_context_patched._ambari_patched = True
ssl._create_default_https_context = _create_default_https_context_patched
+
+"""
+See RFC3986, Appendix B
+Tested on the following cases:
+ "192.168.54.1"
+ "192.168.54.2:7661
+ "hdfs://192.168.54.3/foo/bar"
+ "ftp://192.168.54.4:7842/foo/bar"
+
+ Returns None if only a port is passed in
+"""
+def get_host_from_url(uri):
+ if uri is None:
+ return None
+
+ # if not a string, return None
+ if not isinstance(uri, basestring):
+ return None
+
+ # RFC3986, Appendix B
+ parts = re.findall('^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?', uri)
+
+ # index of parts
+ # scheme = 1
+ # authority = 3
+ # path = 4
+ # query = 6
+ # fragment = 8
+
+ host_and_port = uri
+ if 0 == len(parts[0][1]):
+ host_and_port = parts[0][4]
+ elif 0 == len(parts[0][2]):
+ host_and_port = parts[0][1]
+ elif parts[0][2].startswith("//"):
+ host_and_port = parts[0][3]
+
+ if -1 == host_and_port.find(':'):
+ if host_and_port.isdigit():
+ return None
+
+ return host_and_port
+ else:
+ return host_and_port.split(':')[0]
+
diff --git a/contrib/management-packs/isilon-onefs-mpack/pom.xml b/contrib/management-packs/isilon-onefs-mpack/pom.xml
index 5d8f215..6d5411c 100644
--- a/contrib/management-packs/isilon-onefs-mpack/pom.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/pom.xml
@@ -27,6 +27,7 @@
<minAmbariVersion>3.0.0.0</minAmbariVersion>
<maxAmbariVersion></maxAmbariVersion>
<nifiversion>1.0.0</nifiversion>
+ <custom.tests>false</custom.tests>
</properties>
<parent>
<groupId>org.apache.ambari.contrib.mpacks</groupId>
@@ -88,6 +89,28 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <configuration>
+ <executable>python</executable>
+ <workingDirectory>src/test/python</workingDirectory>
+ <arguments>
+ <argument>unitTests.py</argument>
+ <argument>${custom.tests}</argument>
+ </arguments>
+ </configuration>
+ <id>python-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<tarLongFileMode>gnu</tarLongFileMode>
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
index 7d3acd7..e2a7922 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/core-site.xml
@@ -27,6 +27,12 @@
literal string "local" or a host:port for HDFS.</description>
<final>true</final>
<on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>onefs</type>
+ <name>onefs_host</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>hadoop.security.authentication</name>
@@ -91,12 +97,4 @@ DEFAULT
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
- <property>
- <name>net.topology.script.file.name</name>
- <value>/etc/hadoop/conf/topology_script.py</value>
- <description>
- Location of topology script used by Hadoop to determine the rack location of nodes.
- </description>
- <on-ambari-upgrade add="true"/>
- </property>
</configuration>
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
index cb6544f..727f450 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/hdfs-site.xml
@@ -31,26 +31,30 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:8082</value>
- <description>
- The datanode http server address and port.
- </description>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
<name>dfs.namenode.http-address</name>
<value>localhost:8082</value>
<description>The name of the default file system. Either the
literal string "local" or a host:port for HDFS.</description>
<final>true</final>
- <on-ambari-upgrade add="false"/>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>onefs</type>
+ <name>onefs_host</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>dfs.namenode.https-address</name>
<value>localhost:8080</value>
<description>The https address where namenode binds</description>
- <on-ambari-upgrade add="false"/>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>onefs</type>
+ <name>onefs_host</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>dfs.client-write-packet-size</name>
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml
new file mode 100644
index 0000000..bcbf2c8
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/configuration/onefs.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>onefs_host</name>
+ <value/>
+ <description>SmartConnect Zone Address of OneFS</description>
+ <display-name>OneFS Host</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
index 5ba562e..06a90d9 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/metainfo.xml
@@ -41,6 +41,11 @@
<configFiles>
<configFile>
<type>xml</type>
+ <fileName>onefs.xml</fileName>
+ <dictionaryName>onefs</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>xml</type>
<fileName>hdfs-site.xml</fileName>
<dictionaryName>hdfs-site</dictionaryName>
</configFile>
@@ -80,9 +85,17 @@
<timeout>300</timeout>
</commandScript>
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
<configuration-dependencies>
<config-type>core-site</config-type>
<config-type>hdfs-site</config-type>
+ <config-type>onefs</config-type>
<config-type>hadoop-env</config-type>
</configuration-dependencies>
<restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
index 953efdc..79aff8f 100644
--- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py
@@ -17,15 +17,14 @@ limitations under the License.
"""
-from resource_management.libraries.functions.version import format_stack_version, compare_versions
-from resource_management import *
-import os
-import itertools
-import re
+from resource_management import Script
from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format
config = Script.get_config()
@@ -45,6 +44,9 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
+java64_home = config['hostLevelParams']['java_home']
+java_exec = format("{java64_home}/bin/java")
+
import functools
#create partial functions with common arguments for every HdfsResource call
#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py
new file mode 100644
index 0000000..630ff62
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/service_advisor.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+from ambari_commons import inet_utils
+import imp
+import traceback
+
+def error(message): return {"level": "ERROR", "message": message}
+
+class Uri:
+ @classmethod
+ def default_fs(self, configs):
+ return self.from_config(configs, 'core-site', 'fs.defaultFS')
+
+ @classmethod
+ def http_namenode(self, configs):
+ return self.from_config(configs, 'hdfs-site', 'dfs.namenode.http-address')
+
+ @classmethod
+ def https_namenode(self, configs):
+ return self.from_config(configs, 'hdfs-site', 'dfs.namenode.https-address')
+
+ @classmethod
+ def onefs(self, configs):
+ return self.from_config(configs, 'onefs', 'onefs_host')
+
+ @staticmethod
+ def from_config(configs, config_type, property_name):
+ return Uri(configs['configurations'][config_type]['properties'][property_name])
+
+ def __init__(self, address):
+ self.address = address
+
+ def has_host(self, uri):
+ return uri.hostname() == self.hostname()
+
+ def hostname(self):
+ return inet_utils.get_host_from_url(self.address)
+
+ def fix_host(self, uri):
+ if not uri.hostname() or not self.hostname():
+ return self.address
+ return self.address.replace(self.hostname(), uri.hostname())
+
+ def __str__(self):
+ return self.address
+
+class CoreSite:
+ def __init__(self, configs):
+ self.configs = configs
+
+ def validate(self):
+ invalid_configs = []
+ onefs_host = Uri.onefs(self.configs)
+ if not Uri.default_fs(self.configs).has_host(onefs_host):
+ invalid_configs.append({
+ 'config-name': 'fs.defaultFS',
+ 'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+ })
+ return invalid_configs
+
+class HdfsSite:
+ def __init__(self, configs):
+ self.configs = configs
+
+ def validate(self):
+ invalid_configs = []
+ onefs_host = Uri.onefs(self.configs)
+ if not Uri.http_namenode(self.configs).has_host(onefs_host):
+ invalid_configs.append({
+ 'config-name': 'dfs.namenode.http-address',
+ 'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+ })
+ if not Uri.https_namenode(self.configs).has_host(onefs_host):
+ invalid_configs.append({
+ 'config-name': 'dfs.namenode.https-address',
+ 'item': error('Hostname should match OneFS host: {0}'.format(onefs_host))
+ })
+ return invalid_configs
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+else:
+ class HDP26ONEFSServiceAdvisor(service_advisor.ServiceAdvisor):
+ def getServiceConfigurationRecommendations(self, configs, clusterData, services, hosts):
+ putCoreSiteProperty = self.putProperty(configs, "core-site", services)
+ putHdfsSiteProperty = self.putProperty(configs, "hdfs-site", services)
+ onefs_host = Uri.onefs(services)
+ putCoreSiteProperty("fs.defaultFS", Uri.default_fs(services).fix_host(onefs_host))
+ putHdfsSiteProperty("dfs.namenode.http-address", Uri.http_namenode(services).fix_host(onefs_host))
+ putHdfsSiteProperty("dfs.namenode.https-address", Uri.https_namenode(services).fix_host(onefs_host))
+
+ def getServiceConfigurationsValidationItems(self, configs, recommendedDefaults, services, hosts):
+ validation_errors = []
+ validation_errors.extend(self.toConfigurationValidationProblems(CoreSite(services).validate(), 'core-site'))
+ validation_errors.extend(self.toConfigurationValidationProblems(HdfsSite(services).validate(), 'hdfs-site'))
+ return validation_errors
+
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json
new file mode 100644
index 0000000..8301bf0
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/themes/theme.json
@@ -0,0 +1,60 @@
+{
+ "name": "default",
+ "description": "Default theme for OneFS service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-onefs",
+ "display-name": "OneFS",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-onefs-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "onefs/onefs_host",
+ "subsection-name": "subsection-onefs-col1"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "onefs/onefs_host",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
+
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py b/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py
new file mode 100644
index 0000000..26714b3
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/test/python/TestServiceAdvisor.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from unittest import TestCase
+from service_advisor import Uri, CoreSite, HdfsSite
+
+configs = {
+ 'configurations': {
+ 'core-site': {
+ 'properties': {
+ 'fs.defaultFS': 'hdfs://localhost:8020'
+ }
+ },
+ 'hdfs-site': {
+ 'properties': {
+ 'dfs.namenode.http-address' : 'scisilon.fqdn:8082',
+ 'dfs.namenode.https-address' : 'scisilon.fqdn:8080',
+ }
+ },
+ 'onefs': {
+ 'properties': {
+ 'onefs_host': 'scisilon.fqdn'
+ }
+ }
+ }
+}
+
+class TestUri(TestCase):
+ def test_fix_host(self):
+ onefs_host = Uri.onefs(configs)
+ default_fs = Uri.default_fs(configs)
+ fixed = default_fs.fix_host(onefs_host)
+ self.assertEquals('hdfs://scisilon.fqdn:8020', fixed)
+
+ def test_skip_replacing_to_empty_host(self):
+ default_fs = Uri.default_fs(configs)
+ self.assertEquals('hdfs://localhost:8020', default_fs.fix_host(Uri("")))
+
+ def test_skip_fixing_invalid_host(self):
+ default_fs = Uri("hdfs://:8080")
+ self.assertEquals('hdfs://:8080', default_fs.fix_host(Uri("host")))
+
+ def test_core_site_validation_error_on_host_mismatch(self):
+ core_site = CoreSite(configs)
+ erros = core_site.validate()
+ self.assertEquals(len(erros), 1)
+ self.assertEquals(erros[0]['config-name'], 'fs.defaultFS')
+
+ def test_hdfs_site_no_validation_error(self):
+ hdfs_site = HdfsSite(configs)
+ erros = hdfs_site.validate()
+ self.assertEquals(len(erros), 0)
\ No newline at end of file
diff --git a/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py b/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py
new file mode 100644
index 0000000..03bc923
--- /dev/null
+++ b/contrib/management-packs/isilon-onefs-mpack/src/test/python/unitTests.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import unittest
+import os
+import sys
+from random import shuffle
+import fnmatch
+
+#excluded directories with non-test staff from stack and service scanning,
+#also we can add service or stack to skip here
+STACK_EXCLUDE = ["utils"]
+SERVICE_EXCLUDE = ["configs"]
+
+TEST_MASK = '[Tt]est*.py'
+CUSTOM_TEST_MASK = '_[Tt]est*.py'
+def get_parent_path(base, directory_name):
+ """
+ Returns absolute path for directory_name, if directory_name present in base.
+ For example, base=/home/user/test2, directory_name=user - will return /home/user
+ """
+ done = False
+ while not done:
+ base = os.path.dirname(base)
+ if base == "/":
+ return None
+ if os.path.split(base)[-1] == directory_name:
+ done = True
+ else:
+ done = False
+ return base
+
+def get_test_files(path, mask = None, recursive=True):
+ """
+ Returns test files for path recursively
+ """
+ current = []
+ directory_items = os.listdir(path)
+ directory_items.sort()
+
+ for item in directory_items:
+ add_to_pythonpath = False
+ item_path = os.path.join(path, item)
+ if os.path.isfile(item_path):
+ if fnmatch.fnmatch(item, mask):
+ add_to_pythonpath = True
+ current.append(item)
+ elif os.path.isdir(item_path):
+ if recursive:
+ current.extend(get_test_files(item_path, mask = mask))
+ if add_to_pythonpath:
+ sys.path.append(path)
+ return current
+
+
+def main():
+ custom_tests = False
+ if len(sys.argv) > 1:
+ if sys.argv[1] == "true":
+ custom_tests = True
+ pwd = os.path.abspath(os.path.dirname(__file__))
+
+ project_folder = get_parent_path(pwd,'isilon-onefs-mpack')
+ sys.path.append(project_folder + "/src/main/resources/addon-services/ONEFS/1.0.0")
+ sys.path.append(project_folder + "/src/test/python")
+ sys.path.append(project_folder + "/../../../ambari-agent/src/main/python")
+ sys.path.append(project_folder + "/../../../ambari-common/src/main/python")
+
+ has_failures = False
+ test_runs = 0
+ test_failures = []
+ test_errors = []
+ sys.stderr.write("Running tests\n")
+ if custom_tests:
+ test_mask = CUSTOM_TEST_MASK
+ else:
+ test_mask = TEST_MASK
+
+ tests = get_test_files(pwd, mask=test_mask, recursive=True)
+ shuffle(tests)
+ modules = [os.path.basename(s)[:-3] for s in tests]
+ suites = [unittest.defaultTestLoader.loadTestsFromName(name) for name in
+ modules]
+ testSuite = unittest.TestSuite(suites)
+ textRunner = unittest.TextTestRunner(verbosity=2).run(testSuite)
+ test_runs += textRunner.testsRun
+ test_errors.extend([(str(item[0]),str(item[1]),"ERROR") for item in textRunner.errors])
+ test_failures.extend([(str(item[0]),str(item[1]),"FAIL") for item in textRunner.failures])
+ tests_status = textRunner.wasSuccessful() and not has_failures
+
+ if not tests_status:
+ sys.stderr.write("----------------------------------------------------------------------\n")
+ sys.stderr.write("Failed tests:\n")
+ for failed_tests in [test_errors,test_failures]:
+ for err in failed_tests:
+ sys.stderr.write("{0}: {1}\n".format(err[2],err[0]))
+ sys.stderr.write("----------------------------------------------------------------------\n")
+ sys.stderr.write("{0}\n".format(err[1]))
+ sys.stderr.write("----------------------------------------------------------------------\n")
+ sys.stderr.write("Total run:{0}\n".format(test_runs))
+ sys.stderr.write("Total errors:{0}\n".format(len(test_errors)))
+ sys.stderr.write("Total failures:{0}\n".format(len(test_failures)))
+
+ if tests_status:
+ sys.stderr.write("OK\n")
+ exit_code = 0
+ else:
+ sys.stderr.write("ERROR\n")
+ exit_code = 1
+ return exit_code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+
--
To stop receiving notification emails like this one, please contact
"commits@ambari.apache.org" <co...@ambari.apache.org>.