You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ra...@apache.org on 2012/05/18 00:03:20 UTC
svn commit: r1339881 - in /incubator/ambari/branches/ambari-186: ./
hmc/ShellScripts/ hmc/db/ hmc/html/ hmc/js/ hmc/package/rpm/SOURCES/
hmc/package/rpm/SPECS/ hmc/php/frontend/ hmc/php/frontend/addNodes/
hmc/php/util/
Author: ramya
Date: Thu May 17 22:03:19 2012
New Revision: 1339881
URL: http://svn.apache.org/viewvc?rev=1339881&view=rev
Log:
AMBARI-271. Support for local yum mirror. Contributed by Hitesh
Added:
incubator/ambari/branches/ambari-186/hmc/package/rpm/SOURCES/hmc_hdp.repo
Modified:
incubator/ambari/branches/ambari-186/CHANGES.txt
incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh
incubator/ambari/branches/ambari-186/hmc/db/schema.dump
incubator/ambari/branches/ambari-186/hmc/html/initializeCluster.php
incubator/ambari/branches/ambari-186/hmc/js/addNodes.js
incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec
incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/bootstrap.php
incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php
incubator/ambari/branches/ambari-186/hmc/php/frontend/createCluster.php
incubator/ambari/branches/ambari-186/hmc/php/frontend/nodesAction.php
incubator/ambari/branches/ambari-186/hmc/php/util/YumRepoConfigParser.php
Modified: incubator/ambari/branches/ambari-186/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/CHANGES.txt?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/CHANGES.txt (original)
+++ incubator/ambari/branches/ambari-186/CHANGES.txt Thu May 17 22:03:19 2012
@@ -2,6 +2,8 @@ Ambari Change log
Release 0.x.x - unreleased
+ AMBARI-271. Support for local yum mirror (Hitesh via ramya)
+
AMBARI-270. Puppet cleanup to define all the users in a common
location (ramya)
Modified: incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh (original)
+++ incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh Thu May 17 22:03:19 2012
@@ -25,47 +25,120 @@
# * - list of hosts
# */
#set -e
-#set -x
+#set -x
trap 'pp_cmd=$ppp_cmd; ppp_cmd=$previous_command; previous_command=$this_command; this_command=$BASH_COMMAND' DEBUG
#trap 'echo "$host: retcode:[$?] command:[$previous_command], out:[$out]"' EXIT
#printf 'Argument is __%s__\n' "$@"
-master=$1
-repo_name=$2
-repo_desc=$3
-repourl=$4
-gpgkeyurl=$5
+usage() {
+ echo "
+Usage: $0 with the following parameters
+ --puppet-master Puppet Master
+ --repo-file Repo File
+ --gpg-key-files GPG Key files - comma-separated
+ "
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'puppet-master:' \
+ -l 'repo-file:' \
+ -l 'gpg-key-files:' \
+ -l 'help' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+ echo "Invalid args" >&2
+ exit 3
+fi
+
+echo "DEBUG: opts ${OPTS}"
+
+eval set -- "${OPTS}"
+while true ; do
+ case "$1" in
+ --puppet-master)
+ MASTER=$2 ; shift 2
+ ;;
+ --repo-file)
+ REPOFILE=$2 ; shift 2
+ ;;
+ --gpg-key-files)
+ GPGKEYFILESTR=$2 ; shift 2
+ ;;
+ --help)
+ usage ;
+ exit 0
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1" >&2
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+if [[ "x" == "x${MASTER}" ]]; then
+ echo "Error: Puppet master not specified" >&2
+ exit 3
+fi
+
+if [[ "x" == "x${REPOFILE}" ]]; then
+ echo "Error: Repo file not specified" >&2
+ exit 3
+fi
+
+if [[ "x" != "x${GPGKEYFILESTR}" ]]; then
+ GPGKEYFILES=$(echo ${GPGKEYFILESTR} | tr "," " ")
+fi
+
+master=${MASTER}
+repoFile=${REPOFILE}
+gpgKeyFiles=${GPGKEYFILES}
+
+echo "DEBUG: Puppet Master: ${master}"
+echo "DEBUG: Repo File: ${repoFile}"
+echo "DEBUG: GPG Key File Locations: ${gpgKeyFiles}"
+
+if [[ ! -f ${repoFile} ]]; then
+ echo "Error: Repo file ${repoFile} does not exist" >&2
+ exit 3
+else
+ echo "Copying $repoFile to /etc/yum.repos.d/"
+ cp -f $repoFile /etc/yum.repos.d/
+fi
+
+repoFileName=`basename $repoFile`
+if [[ ! -f "/etc/yum.repos.d/${repoFileName}" ]]; then
+ echo "Error: Repo file ${repoFile} not copied over to /etc/yum.repos.d/" >&2
+ exit 3
+fi
+
+for gpgKeyFile in ${gpgKeyFiles}
+do
+ if [[ ! -f ${gpgKeyFile} ]]; then
+ echo "Error: Specified GPG key file ${gpgKeyFile} does not exist" >&2
+ exit 3
+ fi
+ echo "Copying ${gpgKeyFile} to /etc/pki/rpm-gpg/"
+ cp -f ${gpgKeyFile} /etc/pki/rpm-gpg/
+ gpgKeyFileName=`basename ${gpgKeyFile}`
+ if [[ ! -f "/etc/pki/rpm-gpg/${gpgKeyFileName}" ]]; then
+ echo "Error: GPG key file ${gpgKeyFile} not copied over to /etc/pki/rpm-gpg/" >&2
+ exit 3
+ fi
+done
host=`hostname -f | tr '[:upper:]' '[:lower:]'`
-#echo "$host:_ERROR_:$master, $repo_name, $repo_desc, $repourl, $gpgkeyurl"
-
-repo_file_content=''
-if [[ -z "$gpgkeyurl" ]]; then
- repo_file_content="[$repo_name]\nname=$repo_desc\nbaseurl=$repourl\nenabled=1\ngpgcheck=0"
-else
- repo_file_content="[$repo_name]\nname=$repo_desc\nbaseurl=$repourl\nenabled=1\ngpgcheck=1\ngpgkey=$gpgkeyurl"
-fi
-
-out=`echo -e $repo_file_content > /etc/yum.repos.d/$repo_name.repo`
-ret=$?
-if [[ "$ret" != "0" ]]; then
- echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
- exit 1
-fi
-
-
-if [[ ! -z "$gpgkeyurl" ]]; then
- out=`rpm --import $gpgkeyurl`
- ret=$?
- if [[ "$ret" != "0" ]]; then
- echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
- exit 1
- fi
-fi
-
-out=`rpm -Uvh http://dl.fedoraproject.org/pub/epel/5/i386/epel-release-5-4.noarch.rpm 2>&1`
out=`/etc/init.d/iptables stop 1>/dev/null`
+
+echo "Installing puppet using yum"
out=`yum install -y puppet-2.7.9-2`
ret=$?
if [[ "$ret" != "0" ]]; then
@@ -76,15 +149,20 @@ out=`mkdir -p /etc/puppet/agent 2>&1`
agent_auth_conf="path /run\nauth any\nallow $master\n\npath /\nauth any"
out=`echo -e $agent_auth_conf > /etc/puppet/agent/auth.conf`
out=`touch /etc/puppet/agent/namespaceauth.conf`
+
out=`cp -f /etc/puppet/puppet.conf /etc/puppet/agent/ 2>&1`
ret=$?
if [[ "$ret" != "0" ]]; then
echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
exit 1
fi
+
#TODO clean this up for better fix. For now make sure we stop puppet agent. The issue here is we do not know if we started this puppet agent during our run or not.
+echo "Stopping puppet agent using service stop command"
out=`service puppet stop`
ret=$?
+
+echo "Starting puppet agent for HMC"
out=`puppet agent --verbose --confdir=/etc/puppet/agent --listen --runinterval 5 --server $master --report --no-client --waitforcert 10 --configtimeout 600 --debug --logdest=/var/log/puppet_agent.log --httplog /var/log/puppet_agent_http.log --autoflush 2>&1`
ret=$?
if [[ "$ret" != "0" ]]; then
Modified: incubator/ambari/branches/ambari-186/hmc/db/schema.dump
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/db/schema.dump?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/db/schema.dump (original)
+++ incubator/ambari/branches/ambari-186/hmc/db/schema.dump Thu May 17 22:03:19 2012
@@ -256,6 +256,11 @@ INSERT OR REPLACE INTO "ConfigProperties
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "nagios_web_password", "admin", "Nagios Admin password", "", "NAGIOS", "SECRET", '{ "isPassword": true, "noDisplay": false, "reconfigurable": false, "displayType": "text" }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "nagios_contact", "", "Nagios Admin email", "", "NAGIOS", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text" }' );
+-- Configuration for local yum mirror support
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "yum_repo_file", "/usr/share/hmc/yum_repo/hmc_hdp.repo", "Path to YUM Repo file", "Path to YUM Repo file", "MISCELLANEOUS", "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "apache_artifacts_download_url", "", "Apache Artifacts Download URL", "URL form where to download HDP artifacts", "MISCELLANEOUS", "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "gpl_artifacts_download_url", "", "GPL Artifacts Download URL", "URL form where to download GPL artifacts", "MISCELLANEOUS", "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+
-- gsCluster.properties keys
-- maps to hadoop_heap_size in gsCluster.properties in MB
Modified: incubator/ambari/branches/ambari-186/hmc/html/initializeCluster.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/html/initializeCluster.php?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/html/initializeCluster.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/html/initializeCluster.php Thu May 17 22:03:19 2012
@@ -7,7 +7,7 @@
<link type="text/css" rel="stylesheet" href="../css/bootstrap.css" media="screen"/>
<link type="text/css" rel="stylesheet" href="../css/common.css" media="screen"/>
<link type="text/css" rel="stylesheet" href="../css/common2.css" media="screen"/>
- <link type="text/css" rel="stylesheet" href="../css/common3.css" media="screen"/>
+ <link type="text/css" rel="stylesheet" href="../css/common3.css" media="screen"/>
<link type="text/css" rel="stylesheet" href="../css/selectHosts.css" media="screen"/>
<!-- End CSS -->
</head>
@@ -128,7 +128,31 @@
<label for="clusterHostsFileId">Newline-delimited list of node hostnames</label>
<input type="file" name="clusterHostsFile" id="clusterHostsFileId" value="" placeholder="">
</p>
-
+ <br/>
+ <div id="yumMirrorSupportFormId">
+ <div id="yumMirrorSupportFormButtonWrapperId">
+ <p>
+ <label for="yumMirrorSupportFormButtonId">Use local yum mirror instead of HMC defaults?</label>
+ <input type="checkbox" name="YumMirrorSupportFormButton" id="yumMirrorSupportFormButtonId" value="" placeholder="">
+ </p>
+ </div>
+ <div id="yumMirrorSupportFormFieldsId" style="display:none">
+ <p>
+ <label for="yumRepoFilePathId">YUM Repo File Path</label>
+ <input type="text" name="YumRepoFilePath" id="yumRepoFilePathId" value="" placeholder="">
+ </p>
+ <br/>
+ <p>
+ <label for="hmcArtifactsDownloadUrlId">URL from where to download Apache Artifacts</label>
+ <input type="text" name="HmcArtifactsDownloadUrl" id="hmcArtifactsDownloadUrlId" value="" placeholder="">
+ </p>
+ <br/>
+ <p>
+ <label for="hmcGplArtifactsDownloadUrlId">URL from where to download GPL Artifacts</label>
+ <input type="text" name="HmcGplArtifactsDownloadUrl" id="hmcGplArtifactsDownloadUrlId" value="" placeholder="">
+ </p>
+ </div>
+ </div>
<div id="fileUploadWrapperDivId">
<iframe name="fileUploadTarget" id="fileUploadTargetId" src="about:blank" style="display:none"></iframe>
</div>
Modified: incubator/ambari/branches/ambari-186/hmc/js/addNodes.js
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/js/addNodes.js?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/js/addNodes.js (original)
+++ incubator/ambari/branches/ambari-186/hmc/js/addNodes.js Thu May 17 22:03:19 2012
@@ -3,19 +3,40 @@ InstallationWizard.AddNodes = {
renderData:
{},
- render:
+ render:
function (addNodesRenderData) {
/* Always update the object's renderData first. */
InstallationWizard.AddNodes.renderData = addNodesRenderData;
- /* Since this screen is completely statically rendered, nothing else
- * needs to be done here.
+ /* Since this screen is completely statically rendered, nothing else
+ * needs to be done here.
*/
+ if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+ globalYui.one("#yumRepoFilePathId").set('value', '');
+ globalYui.one("#hmcArtifactsDownloadUrlId").set('value', '');
+ globalYui.one("#hmcGplArtifactsDownloadUrlId").set('value', '');
+ }
globalYui.one("#addNodesCoreDivId").setStyle('display', 'block');
+
+ if (globalYui.one("#yumMirrorSupportFormButtonId") && addNodesRenderData.yumRepo) {
+ globalYui.one("#yumRepoFilePathId").set('value', addNodesRenderData.yumRepo.yumRepoFilePath);
+ globalYui.one("#hmcArtifactsDownloadUrlId").set('value', addNodesRenderData.yumRepo.hdpArtifactsDownloadUrl);
+ globalYui.one("#hmcGplArtifactsDownloadUrlId").set('value', addNodesRenderData.yumRepo.gplArtifactsDownloadUrl);
+ }
}
};
+if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+ globalYui.one("#yumMirrorSupportFormButtonId").on('click', function(e) {
+ if (globalYui.one("#yumMirrorSupportFormButtonId").get('checked')) {
+ globalYui.one('#yumMirrorSupportFormFieldsId').setStyle('display', 'block');
+ } else {
+ globalYui.one('#yumMirrorSupportFormFieldsId').setStyle('display', 'none');
+ }
+ });
+}
+
globalYui.one('#addNodesSubmitButtonId').on('click',function (e) {
var focusId = '';
@@ -40,7 +61,7 @@ globalYui.one('#addNodesSubmitButtonId')
}
if (message != '') {
message += ',';
- }
+ }
message += 'User Identity file not specified';
globalYui.one("#clusterDeployUserIdentityFileId").addClass('formInputError');
} else {
@@ -55,7 +76,7 @@ globalYui.one('#addNodesSubmitButtonId')
}
if (message != '') {
message += ',';
- }
+ }
message += 'Hosts file not specified';
globalYui.one("#clusterHostsFileId").addClass('formInputError');
} else {
@@ -68,6 +89,51 @@ globalYui.one('#addNodesSubmitButtonId')
return;
}
+ if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+ if (globalYui.one("#yumMirrorSupportFormButtonId").get('checked')) {
+ // local yum mirror support
+ var repoFile = globalYui.Lang.trim(globalYui.one("#yumRepoFilePathId").get('value'));
+ var artifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcArtifactsDownloadUrlId").get('value'));
+ var gplArtifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcGplArtifactsDownloadUrlId").get('value'));
+
+ if (repoFile = '') {
+ errCount++;
+ if (focusId == '') {
+ focusId = '#yumRepoFilePathId';
+ }
+ if (message != '') {
+ message += ',';
+ }
+ message += 'Yum Repo file not specified';
+ globalYui.one("#yumRepoFilePathId").addClass('formInputError');
+ }
+
+ if (artifactsUrl = '') {
+ errCount++;
+ if (focusId == '') {
+ focusId = '#hmcArtifactsDownloadUrlId';
+ }
+ if (message != '') {
+ message += ',';
+ }
+ message += 'HDP Artifacts Download URL not specified';
+ globalYui.one("#hmcArtifactsDownloadUrlId").addClass('formInputError');
+ }
+
+ if (artifactsUrl = '') {
+ errCount++;
+ if (focusId == '') {
+ focusId = '#hmcGplArtifactsDownloadUrlId';
+ }
+ if (message != '') {
+ message += ',';
+ }
+ message += 'GPL Artifacts Download URL not specified';
+ globalYui.one("#hmcGplArtifactsDownloadUrlId").addClass('formInputError');
+ }
+ }
+ }
+
clearFormStatus();
showLoadingImg();
@@ -77,19 +143,19 @@ globalYui.one('#addNodesSubmitButtonId')
var addNodesFilesForm = globalYui.one("#addNodesFilesFormId");
- addNodesFilesForm.set('action', '../php/frontend/addNodes.php?clusterName=' +
+ addNodesFilesForm.set('action', '../php/frontend/addNodes.php?clusterName=' +
InstallationWizard.AddNodes.renderData.clusterName + "&freshInstall=" + InstallationWizard.AddNodes.renderData.freshInstall);
- /* Set the target of the first form's upload to be a hidden iframe
- * on the page so as not to redirect to the PHP page we're POSTing
+ /* Set the target of the first form's upload to be a hidden iframe
+ * on the page so as not to redirect to the PHP page we're POSTing
* to.
*
- * See http://www.openjs.com/articles/ajax/ajax_file_upload/ for
+ * See http://www.openjs.com/articles/ajax/ajax_file_upload/ for
* more on this.
*/
addNodesFilesForm.set('target', 'fileUploadTarget');
- /* And then programmatically submit the first of the 2 forms. */
+ /* And then programmatically submit the first of the 2 forms. */
addNodesFilesForm.submit();
globalYui.log("Files submitted to server.");
@@ -102,8 +168,24 @@ globalYui.one("#fileUploadTargetId").on(
globalYui.log("File upload finished");
+ var repoFile = '';
+ var artifactsUrl = '';
+ var gplArtifactsUrl = '';
+
+ if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+ if (globalYui.one("#yumMirrorSupportFormButtonId").get('checked')) {
+ // local yum mirror support
+ repoFile = globalYui.Lang.trim(globalYui.one("#yumRepoFilePathId").get('value'));
+ artifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcArtifactsDownloadUrlId").get('value'));
+ gplArtifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcGplArtifactsDownloadUrlId").get('value'));
+ }
+ }
+
var addNodesRequestData = {
- "ClusterDeployUser" : globalYui.Lang.trim(globalYui.one("#clusterDeployUserId").get('value'))
+ "ClusterDeployUser" : globalYui.Lang.trim(globalYui.one("#clusterDeployUserId").get('value')),
+ "yumRepoFilePath": repoFile,
+ "hdpArtifactsDownloadUrl" : artifactsUrl,
+ "gplArtifactsDownloadUrl": gplArtifactsUrl
}
// Trigger the execution of setting up nodes
@@ -125,7 +207,7 @@ globalYui.one("#fileUploadTargetId").on(
globalYui.log("PARSED DATA: " + globalYui.Lang.dump(setupNodesJson));
if (setupNodesJson.result != 0) {
// Error!
- alert("Got error!" + setupNodesJson.error);
+ alert("Got error!" + setupNodesJson.error);
return;
}
setupNodesJson = setupNodesJson.response;
Added: incubator/ambari/branches/ambari-186/hmc/package/rpm/SOURCES/hmc_hdp.repo
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/package/rpm/SOURCES/hmc_hdp.repo?rev=1339881&view=auto
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/package/rpm/SOURCES/hmc_hdp.repo (added)
+++ incubator/ambari/branches/ambari-186/hmc/package/rpm/SOURCES/hmc_hdp.repo Thu May 17 22:03:19 2012
@@ -0,0 +1,14 @@
+[epel]
+name=Extra Packages for Enterprise Linux 5 - $basearch
+#baseurl=http://download.fedoraproject.org/pub/epel/5/$basearch
+mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-5&arch=$basearch
+failovermethod=priority
+enabled=1
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL
+
+[Puppet]
+name=Puppet
+baseurl = http://yum.puppetlabs.com/el/5/products/x86_64/
+enabled=1
+gpgcheck=0
Modified: incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec (original)
+++ incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec Thu May 17 22:03:19 2012
@@ -19,19 +19,20 @@
# */
#
-# RPM Spec file for HMC
+# RPM Spec file for HMC
#
Summary: HMC
Name: hmc
Version: 0.0.1
URL: http://hortonworks.com
-Release: 2%{?dist}
+Release: 2%{?dist}
License: Apache License, Version 2.0
Vendor: Hortonworks <hm...@hortonworks.com>
Group: System Environment/Base
Source: %{name}-%{version}.tar.gz
Source1: hmc.init.in
+Source2: hmc_hdp.repo
Buildroot: %{_tmppath}/%{name}-%{version}-buildroot
Requires: php >= 5, sqlite >= 3, php-pdo, php-pecl-json, httpd, puppet = 2.7.9, pdsh, httpd-devel, ruby-devel, rubygems, mod_passenger, mod_ssl
%define web_prefixdir %{_prefix}/share/hmc
@@ -57,6 +58,7 @@ This package provides a Management Conso
%__mkdir -p $RPM_BUILD_ROOT/usr/lib/ruby/site_ruby/1.8/puppet/reports/
%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/
%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/bin/
+%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
%__mkdir -p $RPM_BUILD_ROOT/%{puppet_dir}/
%__mkdir -p $RPM_BUILD_ROOT/%{puppet_dir}/manifests
%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/
@@ -80,6 +82,7 @@ This package provides a Management Conso
%__cp -f yuiCombinator.php $RPM_BUILD_ROOT/%{web_prefixdir}/
%__cp -rf conf $RPM_BUILD_ROOT/%{web_prefixdir}/
%__cp -rf puppet/modules $RPM_BUILD_ROOT/%{puppet_dir}
+%__cp -f "%{SOURCE2}" $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
%__install -D -m0755 puppet/reports/get_revision $RPM_BUILD_ROOT/%{web_prefixdir}/bin
%__cp -rf puppet/reports/hmcreport.rb $RPM_BUILD_ROOT/usr/lib/ruby/site_ruby/1.8/puppet/reports/
echo "Alias /hdp %{_prefix}/share/hdp" > $RPM_BUILD_ROOT/%{httpd_confdir}/hdp_mon_dashboard.conf
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/bootstrap.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/bootstrap.php?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/bootstrap.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/bootstrap.php Thu May 17 22:03:19 2012
@@ -9,6 +9,7 @@ include_once '../db/HMCDBAccessor.php';
include_once "../util/HMCTxnUtils.php";
include_once 'commandUtils.php';
+include_once "../util/YumRepoConfigParser.php";
$logger = new HMCLogger("BootStrap");
$dbAccessor = new HMCDBAccessor($GLOBALS["DB_PATH"]);
@@ -18,24 +19,45 @@ include_once 'commandUtils.php';
global $logger, $dbAccessoar, $hosts, $readFromFile;
$master=strtolower(exec('hostname -f'));
- $repo_name = $repo['name'];
- $repo_desc = $repo['desc'];
- $repo_url = $repo['url'];
- $repo_gpgkey = $repo['gpgkeyurl'];
+ $repoFile = $repo['yumRepoFilePath'];
+ $gpgKeyFiles = $repo['gpgKeyFiles'];
+
exec ("/etc/init.d/iptables stop");
$logger->log_debug("List of hosts to BootStrap ".json_encode($hosts));
$logger->log_debug("Run script for pdsh ".$rscript);
$scpCmd = "scp -o StrictHostKeyChecking=no ";
+
foreach ($hosts as $host) {
$host = trim($host);
- $cmd = "$scpCmd -i $sshkey $rscript $user@$host:/tmp";
- exec ("$scpCmd -i $sshkey $rscript $user@$host:/tmp");
+
+ $filesToCopy = array_merge ( array ($rscript, $repoFile), $gpgKeyFiles);
+
+ /* Copy puppet run script to all nodes */
+ // Copy repo file to each node
+ // Copy gpg keys to each node
+ if (!empty($filesToCopy)) {
+ $cmd = "$scpCmd -i $sshkey " . implode(" ", $filesToCopy)
+ . " $user@$host:/tmp/ ";
+ $logger->log_debug("Running scp command $cmd");
+ exec($cmd);
+ }
}
- /* Copy puppet run script to all nodes */
- $logger->log_debug("/tmp/puppet_agent_install.sh $master $repo_name $repo_desc $repo_url $repo_gpgkey");
+ $remoteRepoFilePath = trim("/tmp/" . basename(trim($repoFile)));
+ $remoteGpgKeyPaths = "";
+ foreach ($gpgKeyFiles as $kFile) {
+ $dFile = trim("/tmp/" . basename(trim($kFile)));
+ if ($remoteGpgKeyPaths != "") {
+ $remoteGpgKeyPaths .= ",";
+ }
+ $remoteGpgKeyPaths .= $dFile;
+ }
+
+ $rcmd = "/tmp/puppet_agent_install.sh --puppet-master=" . $master
+ . " --repo-file=" . $remoteRepoFilePath
+ . " --gpg-key-files=" . $remoteGpgKeyPaths;
+ $logger->log_info("Running $rcmd to bootstrap each node");
- $rcmd = "/tmp/puppet_agent_install.sh $master $repo_name $repo_desc $repo_url $repo_gpgkey";
runPdsh($clusterName, "bootstrapNodes", $user, $readFromFile, $rcmd);
$result = parseAndUpdateNodeInfo ($clusterName, "bootstrapNodes", $logger);
@@ -79,10 +101,71 @@ $sshkey = getSshKeyFilePath($clusterName
$rscript = realpath("../../ShellScripts/puppet_agent_install.sh");
$repository=array();
-$repository['name']="hmc_puppet";
-$repository['desc']="puppetlabs";
-$repository['url']="http://yum.puppetlabs.com/el/5/products/x86_64/";
-$repository['gpgkeyurl']="http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs";
+$configs = $dbAccessor->getServiceConfig($clusterName);
+if ($configs["result"] != 0) {
+ $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+ $logger->log_error("Got error when trying to retrieve configs from DB");
+ return;
+}
+
+$repoFile = $configs["properties"]["yum_repo_file"];
+$gpgKeyLocations = getEnabledGpgKeyLocations($repoFile);
+if ($gpgKeyLocations === FALSE) {
+ $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+ $logger->log_error("Got error when trying to parse yum repo config");
+ return;
+}
+
+$tmpDir = "/tmp/hmcDownloads-".time()."/";
+$retVal = 0;
+$output = array();
+exec("mkdir -p ".$tmpDir, $output, $retVal);
+if ($retVal != 0) {
+ $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+ $logger->log_error("Got error when trying to create tmp download dir"
+ . ", dir=" . $tmpDir . ", output=" . print_r($output, true));
+ return;
+}
+
+$gpgKeyFiles = array();
+
+foreach ($gpgKeyLocations as $repoId => $gpgInfo) {
+ if (!isset($gpgInfo["gpgkey"])) {
+ continue;
+ }
+ $loc = $gpgInfo["gpgkey"];
+ $logger->log_info("Fetching gpg key for $repoId from location $loc");
+ $info = parse_url($loc);
+ if ($info === FALSE || !isset($info["path"])) {
+ $logger->log_error("Skipping invalid url $loc");
+ continue;
+ }
+ $fileName = basename($info["path"]);
+
+ $destFilePath = $tmpDir . "/" . $fileName;
+
+ $fetchCurlCmd = "curl --connect-timeout 30 --fail -s -o "
+ . $destFilePath . " " . $loc;
+
+ $logger->log_info("Fetching gpg key for $repoId from location $loc using "
+ . $fetchCurlCmd);
+
+ $retVal = 0;
+ $output = array();
+ exec($fetchCurlCmd, $output, $retVal);
+
+ if ($retVal != 0) {
+ $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+ $logger->log_error("Error when trying to download gpg key using "
+ . $fetchCurlCmd . ", output=" . print_r($output, true));
+ return;
+ }
+ array_push($gpgKeyFiles, $destFilePath);
+}
+
+$repository = array( "yumRepoFilePath" => $repoFile,
+ "gpgKeyFiles" => $gpgKeyFiles);
+
$logger->log_debug("BootStrapping with puppet");
$boot_result = bootstrap($clusterName, $deployUser, $rscript,
$sshkey, $repository);
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php Thu May 17 22:03:19 2012
@@ -157,11 +157,13 @@ function sign_and_verify_agent ($hosts,
}
} else {
$logger->log_error("Failed to do puppet kick -ping on host " . $host);
- $output[$host] =
- array ( "discoveryStatus" => "FAILED",
+ if (!isset($output[$host])) {
+ $output[$host] =
+ array ( "discoveryStatus" => "FAILED",
"badHealthReason" => "Puppet kick failed: "
. ", error=" . $err . ", outputLogs="
. implode(";", $out_arr));
+ }
$hostsState[$host] = FALSE;
}
}
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/createCluster.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/createCluster.php?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/createCluster.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/createCluster.php Thu May 17 22:03:19 2012
@@ -123,11 +123,22 @@ if (!is_dir($clusterDir) && !mkdir($clus
return;
}
+$propertiesArr = $dbAccessor->getConfigPropertiesMetaInfo();
+if ($propertiesArr["result"] != 0) {
+ print json_encode(array( "result" => 1, "error" => "Error in config properties meta info"));
+ return;
+}
+
$output = array(
"result" => 0,
"error" => "",
"response" => array(
- "clusterName" => $response["clusterName"]
+ "clusterName" => $response["clusterName"],
+ "yumRepo" => array (
+ "yumRepoFilePath" => $propertiesArr["configs"]["yum_repo_file"]["value"],
+ "hdpArtifactsDownloadUrl" => $propertiesArr["configs"]["apache_artifacts_download_url"]["value"],
+ "gplArtifactsDownloadUrl" => $propertiesArr["configs"]["gpl_artifacts_download_url"]["value"]
+ )
),
);
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/nodesAction.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/nodesAction.php?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/nodesAction.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/nodesAction.php Thu May 17 22:03:19 2012
@@ -16,8 +16,87 @@ $clusterName = $_GET['clusterName'];
$action = $_GET['action'];
$deployUser = $_POST['ClusterDeployUser'];
+$propertiesArr = $dbAccessor->getConfigPropertiesMetaInfo();
+if ($propertiesArr["result"] != 0) {
+ print json_encode(array( "result" => 1, "error" => "Error in config properties meta info"));
+ return;
+}
+
+// Use meta info defaults
+// Override with current svc configs
+// Override with POST params
+
+$repoFilePath = $propertiesArr["configs"]["yum_repo_file"]["value"];
+$hdpArtifactsDownloadUrl = $propertiesArr["configs"]["apache_artifacts_download_url"]["value"];
+$gplArtifactsDownloadUrl = $propertiesArr["configs"]["gpl_artifacts_download_url"]["value"];
+
+$currentConfigs = $dbAccessor->getServiceConfig($clusterName);
+if ($currentConfigs["result"] != 0) {
+ print json_encode(array( "result" => 1, "error" => "Could not get configs from DB"));
+ return;
+}
+
+if (isset($currentConfigs["properties"]["yum_repo_file"])
+ && $currentConfigs["properties"]["yum_repo_file"] != "") {
+ $repoFilePath = $currentConfigs["properties"]["yum_repo_file"];
+}
+
+if (isset($currentConfigs["properties"]["apache_artifacts_download_url"])
+ && $currentConfigs["properties"]["apache_artifacts_download_url"] != "") {
+ $hdpArtifactsDownloadUrl = $currentConfigs["properties"]["apache_artifacts_download_url"];
+}
+
+if (isset($currentConfigs["properties"]["gpl_artifacts_download_url"])
+ && $currentConfigs["properties"]["gpl_artifacts_download_url"] != "") {
+ $gplArtifactsDownloadUrl = $currentConfigs["properties"]["gpl_artifacts_download_url"];
+}
+
+if (isset($_POST['yumRepoFilePath'])
+ && $_POST['yumRepoFilePath'] != "") {
+ $repoFilePath = $_POST['yumRepoFilePath'];
+}
+
+if (isset($_POST['hdpArtifactsDownloadUrl'])
+ && $_POST['hdpArtifactsDownloadUrl'] != "") {
+ $hdpArtifactsDownloadUrl = $_POST['hdpArtifactsDownloadUrl'];
+}
+
+if (isset($_POST['gplArtifactsDownloadUrl'])
+ && $_POST['gplArtifactsDownloadUrl'] != "") {
+ $gplArtifactsDownloadUrl = $_POST['gplArtifactsDownloadUrl'];
+}
+
header("Content-type: application/json");
+if (!file_exists($repoFilePath)) {
+ print (json_encode(array(
+ "result" => 1,
+ "error" => "Invalid repo file path specified"
+ )
+ ));
+ return;
+}
+
+// TODO - error checks for download urls
+/*
+if (parse_url($hdpArtifactsDownloadUrl) === FALSE
+ || parse_url($gplArtifactsDownloadUrl) === FALSE) {
+ print (json_encode(array(
+ "result" => 1,
+ "error" => "Invalid download urls specified")));
+ return;
+}
+*/
+
+$configs = array ( "yum_repo_file" => $repoFilePath,
+ "apache_artifacts_download_url" => $hdpArtifactsDownloadUrl,
+ "gpl_artifacts_download_url" => $gplArtifactsDownloadUrl);
+$dbResponse = $dbAccessor->updateServiceConfigs($clusterName, $configs);
+if ($dbResponse["result"] != 0) {
+ $logger->log_error("Got error while persisting configs: ".$dbResponse["error"]);
+ return $dbResponse;
+}
+
$stagesFiles = "";
if ($action == "addNodes") {
$stagesFile = "./addNodes/stages.php";
Modified: incubator/ambari/branches/ambari-186/hmc/php/util/YumRepoConfigParser.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/util/YumRepoConfigParser.php?rev=1339881&r1=1339880&r2=1339881&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/util/YumRepoConfigParser.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/util/YumRepoConfigParser.php Thu May 17 22:03:19 2012
@@ -18,6 +18,15 @@
* limitations under the License.
*/
+/**
+ * Parse repo file and get all enabled gpg keys
+ * @param string $repoFile
+ * @return mixed
+ * array (
+ * $currentRepoId = array ( "gpgkey" => $currentGpgLocation),
+ * ....
+ * )
+ */
function getEnabledGpgKeyLocations($repoFile) {
$logger = new HMCLogger("YumRepoConfigParser");
@@ -64,6 +73,8 @@ function getEnabledGpgKeyLocations($repo
&& (($currentGpgCheck == -1 && $globalGpgCheck == 1)
|| ($currentGpgCheck == 1))) {
if ($currentGpgLocation != "") {
+ $logger->log_debug("Adding gpgkey $currentGpgLocation for repo"
+ .", id=" . $currentRepoId);
$response[$currentRepoId] = array ( "gpgkey" => $currentGpgLocation);
}
} else if ($currentRepoId != ""
@@ -118,5 +129,4 @@ function getEnabledGpgKeyLocations($repo
return $response;
}
-
?>