You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by sn...@apache.org on 2015/04/21 20:49:31 UTC
incubator-ranger git commit: RANGER-418: add upgrade scripts
Repository: incubator-ranger
Updated Branches:
refs/heads/master 853a932b3 -> 46b5ecc33
RANGER-418: add upgrade scripts
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/46b5ecc3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/46b5ecc3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/46b5ecc3
Branch: refs/heads/master
Commit: 46b5ecc33c02229d4c8721a8d616f2a788fef6f0
Parents: 853a932
Author: sneethiraj <sn...@apache.org>
Authored: Tue Apr 21 12:14:50 2015 -0400
Committer: sneethiraj <sn...@apache.org>
Committed: Tue Apr 21 14:43:32 2015 -0400
----------------------------------------------------------------------
agents-common/scripts/upgrade-plugin.py | 160 +++++++++
agents-common/scripts/upgrade-plugin.sh | 29 ++
.../scripts/ranger-admin-site-template.xml | 223 +++++++++++++
security-admin/scripts/upgrade.sh | 33 ++
security-admin/scripts/upgrade_admin.py | 321 +++++++++++++++++++
src/main/assembly/admin-web.xml | 10 +
src/main/assembly/hbase-agent.xml | 11 +
src/main/assembly/hdfs-agent.xml | 11 +
src/main/assembly/hive-agent.xml | 11 +
src/main/assembly/knox-agent.xml | 11 +
src/main/assembly/storm-agent.xml | 11 +
11 files changed, 831 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/agents-common/scripts/upgrade-plugin.py
----------------------------------------------------------------------
diff --git a/agents-common/scripts/upgrade-plugin.py b/agents-common/scripts/upgrade-plugin.py
new file mode 100755
index 0000000..9c32dd1
--- /dev/null
+++ b/agents-common/scripts/upgrade-plugin.py
@@ -0,0 +1,160 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import xml.etree.ElementTree as ET
+import os,errno,sys
+from os import listdir
+from os.path import isfile, join, dirname
+from urlparse import urlparse
+
+debugLevel = 1
+
+SUPPORTED_COMPONENTS = [ "hdfs", "hive", "hbase", "knox", "storm" ]
+
+#
+# xmlTemplateDirectory: directory where all of the xml templates are kept here
+#
+
+xmlTemplateDirectory = './install/conf.templates/enable'
+
+def showUsage():
+ print "This script must be run with a <componentName> as parameter"
+ print "USAGE: upgrade-plugin.py <componentName>"
+ print " <componentName> could be any one of the following: %s" % (SUPPORTED_COMPONENTS)
+
+if (len(sys.argv) == 1):
+ showUsage()
+ sys.exit(1)
+
+componentName = sys.argv[1]
+
+if (componentName not in SUPPORTED_COMPONENTS):
+ print "Invalid componentName passed as parameter: %s" % (componentName)
+ showUsage()
+ sys.exit(1)
+
+#
+# For hdfs, the componentName is hadoop (for path calculation)
+#
+
+if (componentName == 'hdfs'):
+ configPath = 'hadoop'
+else:
+ configPath = componentName
+
+#
+# configDirectory: where OLD (champlain) configuration exists and NEW (dal) configuration is written to
+#
+configDirectory = '/etc/' + configPath + '/conf'
+
+
+
+def getXMLConfigKeys(xmlFileName):
+ ret = []
+ tree = ET.parse(xmlFileName)
+ root = tree.getroot()
+ for config in root.iter('property'):
+ name = config.find('name').text
+ ret.append(name)
+ return ret
+
+def getXMLConfigMap(xmlFileName):
+ ret = {}
+ tree = ET.parse(xmlFileName)
+ root = tree.getroot()
+ for config in root.iter('property'):
+ name = config.find('name').text
+ val = config.find('value').text
+ ret[name] = val
+ return ret
+
+def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName):
+ tree = ET.parse(xmlTemplateFileName)
+ root = tree.getroot()
+ for config in root.iter('property'):
+ name = config.find('name').text
+ if (name in prop):
+ config.find('value').text = prop[name]
+ tree.write(xmlOutputFileName)
+
+def rewriteConfig(props,newProps):
+ if (debugLevel > 0):
+ for k,v in props.iteritems():
+ print "old config[%s] = [%s]" % (k,v)
+ #
+ # Derived fields
+ #
+ pmUrl = props['xasecure.' + componentName + '.policymgr.url']
+ url = urlparse(pmUrl)
+ restUrl = url[0] + "://" + url[1]
+ serviceName = url[2].split("/")[-1]
+ props['ranger.plugin.' + componentName + '.policy.rest.url'] = restUrl
+ props['ranger.plugin.' + componentName + '.service.name'] = serviceName
+ props['ranger.plugin.' + componentName + '.policy.pollIntervalMs'] = props['xasecure.' + componentName + '.policymgr.url.reloadIntervalInMillis']
+ #props['ranger.plugin.' + componentName + '.policy.rest.ssl.config.file'] = props['y']
+ fileLoc = props['xasecure.' + componentName + '.policymgr.url.laststoredfile']
+ props['ranger.plugin.' + componentName + '.policy.cache.dir'] = dirname(fileLoc)
+ if ( 'xasecure.policymgr.sslconfig.filename' in props ):
+ props['ranger.plugin.' + componentName + '.policy.rest.ssl.config.file'] = props['xasecure.policymgr.sslconfig.filename']
+ else:
+ sslConfigFileName = join(configDirectory,'ranger-policymgr-ssl.xml')
+ props['ranger.plugin.' + componentName + '.policy.rest.ssl.config.file'] = sslConfigFileName
+ #
+ # Fix for KNOX ssl (missing) configuration
+ #
+ if ('xasecure.policymgr.clientssl.keystore.password' not in props):
+ props['xasecure.policymgr.clientssl.keystore.password'] = 'none'
+ if ('xasecure.policymgr.clientssl.truststore.password' not in props):
+ props['xasecure.policymgr.clientssl.truststore.password'] = 'none'
+ if ('xasecure.policymgr.clientssl.keystore.credential.file' not in props):
+ props['xasecure.policymgr.clientssl.keystore.credential.file'] = 'jceks://file/tmp/keystore-' + serviceName + '-ssl.jceks'
+ if ( 'xasecure.policymgr.clientssl.truststore.credential.file' not in props):
+ props['xasecure.policymgr.clientssl.truststore.credential.file'] = 'jceks://file/tmp/keystore-' + serviceName + '-ssl.jceks'
+
+ for fn in listdir(xmlTemplateDirectory):
+ file = join(xmlTemplateDirectory,fn)
+ if isfile(file) and fn.startswith("ranger-") and fn.endswith(".xml") :
+ newConfigFile = join(configDirectory, fn)
+ writeXMLUsingProperties(file, props, newConfigFile)
+
+def main():
+ props = {}
+ newProps = {}
+ foundFiles = []
+ for fn in listdir(configDirectory):
+ file = join(configDirectory,fn)
+ if isfile(file) and fn.startswith("xasecure-") and fn.endswith(".xml") :
+ foundFiles.append(file)
+ r = getXMLConfigMap(file)
+ props.update(r)
+ if (len(foundFiles) == 0):
+ print "INFO: Previous version of ranger is not enabled/configured for component [%s]" % (componentName)
+ sys.exit(0)
+ if (len(foundFiles) != 3):
+ print "ERROR: Expected to find three files matching xasecure-*.xml files under the folder (%s) - found %s" % (configDirectory,foundFiles)
+ sys.exit(1)
+ for fn in listdir(xmlTemplateDirectory):
+ file = join(xmlTemplateDirectory,fn)
+ if isfile(file) and fn.startswith("ranger-") and fn.endswith(".xml") :
+ r = getXMLConfigMap(file)
+ newProps.update(r)
+ newConfigFile = join(configDirectory,fn)
+ if isfile(newConfigFile):
+ print "ERROR: new config file [%s] already exists. Upgrade script can not overwrite an existing config file." % (newConfigFile)
+ sys.exit(1)
+ rewriteConfig(props,newProps)
+
+main()
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/agents-common/scripts/upgrade-plugin.sh
----------------------------------------------------------------------
diff --git a/agents-common/scripts/upgrade-plugin.sh b/agents-common/scripts/upgrade-plugin.sh
new file mode 100755
index 0000000..8a3d7ab
--- /dev/null
+++ b/agents-common/scripts/upgrade-plugin.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+COMPONENT_NAME=`basename $0 | cut -d. -f1 | sed -e 's:^upgrade-::'`
+
+echo "${COMPONENT_NAME}" | grep -- '-plugin' > /dev/null 2>&1
+
+if [ $? -ne 0 ]
+then
+ echo "$0 : is not applicable for component [${COMPONENT_NAME}]. It is applicable only for ranger plugin component; Exiting ..."
+ exit 0
+fi
+
+HCOMPONENT_NAME=`echo ${COMPONENT_NAME} | sed -e 's:-plugin::'`
+
+./upgrade-plugin.py "${HCOMPONENT_NAME}"
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/security-admin/scripts/ranger-admin-site-template.xml
----------------------------------------------------------------------
diff --git a/security-admin/scripts/ranger-admin-site-template.xml b/security-admin/scripts/ranger-admin-site-template.xml
new file mode 100644
index 0000000..2c0462d
--- /dev/null
+++ b/security-admin/scripts/ranger-admin-site-template.xml
@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>ranger.service.host</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.http.enabled</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.http.port</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.shutdown.port</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.shutdown.command</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.https.port</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.https.attrib.ssl.enabled</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.https.attrib.ssl.protocol</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.https.attrib.client.auth</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.https.attrib.keystore.keyalias</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.service.https.attrib.keystore.pass</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.https.attrib.keystore.file</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.accesslog.dateformat</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.accesslog.pattern</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.externalurl</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.contextName</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.showsql</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.env.local</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.dialect</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.driver</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.url</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.user</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.password</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.maxpoolsize</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.minpoolsize</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.initialpoolsize</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.maxidletime</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.maxstatements</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.preferredtestquery</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.idleconnectiontestperiod</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.credential.alias</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.jdbc.credential.provider.path</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.logs.base.dir</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.scheduler.enabled</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.source.type</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.solr.url</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.dialect</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.driver</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.url</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.user</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.password</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.credential.alias</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.jpa.audit.jdbc.credential.provider.path</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.authentication.method</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.url</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.user.dnpattern</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.group.searchbase</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.group.searchfilter</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.group.roleattribute</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.ad.domain</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.ldap.ad.url</name>
+ <value></value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/security-admin/scripts/upgrade.sh
----------------------------------------------------------------------
diff --git a/security-admin/scripts/upgrade.sh b/security-admin/scripts/upgrade.sh
new file mode 100644
index 0000000..a467298
--- /dev/null
+++ b/security-admin/scripts/upgrade.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+#
+# Ranger Admin Upgrade Script
+#
+# This script will generate install configuration based on the current installation and run setup to upgrade schema
+
+./upgrade_admin.py
+if [ $? -eq 0 ]
+then
+ trap 'rm -f ./install.properties ; exit 1' 2 3 15
+ ./setup.sh
+ ec=$?
+ rm -f ./install.properties
+ exit $ec
+else
+ echo "ERROR: unable to complete upgrade-admin.py"
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/security-admin/scripts/upgrade_admin.py
----------------------------------------------------------------------
diff --git a/security-admin/scripts/upgrade_admin.py b/security-admin/scripts/upgrade_admin.py
new file mode 100755
index 0000000..9c2f2dc
--- /dev/null
+++ b/security-admin/scripts/upgrade_admin.py
@@ -0,0 +1,321 @@
+#!/usr/bin/python
+import re
+import StringIO
+import xml.etree.ElementTree as ET
+import ConfigParser
+import os,errno,sys,getopt
+from os import listdir
+from os.path import isfile, join, dirname, basename
+from urlparse import urlparse
+from time import gmtime, strftime, localtime
+import shutil
+
+debugLevel = 1
+generateXML = 0
+installPropFileName = 'install.properties'
+
+tempLibFolder = "./upgrade-temp"
+
+def showUsage():
+ print "upgrade_admin.py [-g] [-h]"
+ print "This script will generate %s based on currently installed ranger (v0.4.*) configuration." % (installPropFileName)
+ print " -g option will generate ranger-admin-site.xml in the current directory."
+ print " -h will display help text."
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:],"gh")
+except getopt.GetoptError:
+ showUsage()
+ sys.exit(2)
+for opt,arg in opts:
+ if (opt == '-g'):
+ generateXML = 1
+ elif (opt == '-h'):
+ showUsage()
+ sys.exit(0)
+#
+# configDirectory: where OLD (champlain) configuration exists and NEW (dal) configuration is written to
+#
+configDirectory = '/etc/ranger/admin/conf'
+rangerJAASDirectoryName = join(configDirectory,'ranger_jaas')
+
+xaSystemPropFile = 'xa_system.properties'
+ldapPropFile = 'xa_ldap.properties'
+rangerJAASPropFile = 'unixauth.properties'
+securityContextFile = 'security-applicationContext.xml'
+webserverConfigFile = 'ranger_webserver.properties'
+
+rangerSiteXMLFile = "ranger-admin-site.xml"
+
+
+#
+# xmlTemplateDirectory: directory where all of the xml templates are kept here
+#
+templateDirectoryName = './templates-upgrade'
+rangerSiteTemplateXMLFile = "ranger-admin-site-template.xml"
+
+#
+# Install Properties To Ranger Properties
+#
+config2xmlMAP = {
+ 'service.host':'ranger.service.host',
+ 'http.enabled':'ranger.service.http.enabled',
+ 'http.service.port':'ranger.service.http.port',
+ 'service.shutdownPort':'ranger.service.shutdown.port',
+ 'service.shutdownCommand':'ranger.service.shutdown.command',
+ 'https.service.port':'ranger.service.https.port',
+ 'https.attrib.SSLEnabled':'ranger.service.https.attrib.ssl.enabled',
+ 'https.attrib.sslProtocol':'ranger.service.https.attrib.ssl.protocol',
+ 'https.attrib.clientAuth':'ranger.service.https.attrib.client.auth',
+ 'https.attrib.keyAlias':'ranger.service.https.attrib.keystore.keyalias',
+ 'https.attrib.keystorePass':'ranger.service.https.attrib.keystore.pass',
+ 'https.attrib.keystoreFile':'ranger.https.attrib.keystore.file',
+ 'accesslog.dateformat':'ranger.accesslog.dateformat',
+ 'accesslog.pattern':'ranger.accesslog.pattern',
+ 'xa.webapp.url.root':'ranger.externalurl',
+ 'xa.webapp.contextName':'ranger.contextName',
+ 'xa.jpa.showsql':'ranger.jpa.showsql',
+ 'xa.env.local':'ranger.env.local',
+ 'jdbc.dialect':'ranger.jpa.jdbc.dialect',
+ 'jdbc.driver':'ranger.jpa.jdbc.driver',
+ 'jdbc.url':'ranger.jpa.jdbc.url',
+ 'jdbc.user':'ranger.jpa.jdbc.user',
+ 'jdbc.password':'ranger.jpa.jdbc.password',
+ 'jdbc.maxPoolSize':'ranger.jpa.jdbc.maxpoolsize',
+ 'jdbc.minPoolSize':'ranger.jpa.jdbc.minpoolsize',
+ 'jdbc.initialPoolSize':'ranger.jpa.jdbc.initialpoolsize',
+ 'jdbc.maxIdleTime':'ranger.jpa.jdbc.maxidletime',
+ 'jdbc.maxStatements':'ranger.jpa.jdbc.maxstatements',
+ 'jdbc.preferredTestQuery':'ranger.jpa.jdbc.preferredtestquery',
+ 'jdbc.idleConnectionTestPeriod':'ranger.jpa.jdbc.idleconnectiontestperiod',
+ 'xaDB.jdbc.credential.alias':'ranger.jpa.jdbc.credential.alias',
+ 'xaDB.jdbc.credential.provider.path':'ranger.jpa.jdbc.credential.provider.path',
+ 'xa.logs.base.dir':'ranger.logs.base.dir',
+ 'xa.scheduler.enabled':'ranger.scheduler.enabled',
+ 'xa.audit.store':'ranger.audit.source.type',
+ 'audit_solr_url':'ranger.solr.url',
+ 'auditDB.jdbc.dialect':'ranger.jpa.audit.jdbc.dialect',
+ 'auditDB.jdbc.driver':'ranger.jpa.audit.jdbc.driver',
+ 'auditDB.jdbc.url':'ranger.jpa.audit.jdbc.url',
+ 'auditDB.jdbc.user':'ranger.jpa.audit.jdbc.user',
+ 'auditDB.jdbc.password':'ranger.jpa.audit.jdbc.password',
+ 'auditDB.jdbc.credential.alias':'ranger.jpa.audit.jdbc.credential.alias',
+ 'auditDB.jdbc.credential.provider.path':'ranger.jpa.audit.jdbc.credential.provider.path',
+ 'authentication_method':'ranger.authentication.method',
+ 'xa_ldap_url':'ranger.ldap.url',
+ 'xa_ldap_userDNpattern':'ranger.ldap.user.dnpattern',
+ 'xa_ldap_groupSearchBase':'ranger.ldap.group.searchbase',
+ 'xa_ldap_groupSearchFilter':'ranger.ldap.group.searchfilter',
+ 'xa_ldap_groupRoleAttribute':'ranger.ldap.group.roleattribute',
+ 'xa_ldap_ad_domain':'ranger.ldap.ad.domain',
+ 'xa_ldap_ad_url':'ranger.ldap.ad.url' }
+
+def archiveFile(originalFileName):
+ archiveDir = dirname(originalFileName)
+ archiveFileName = "." + basename(originalFileName) + "." + (strftime("%d%m%Y%H%M%S", localtime()))
+ movedFileName = join(archiveDir,archiveFileName)
+ print "INFO: moving [%s] to [%s] ......." % (originalFileName,movedFileName)
+ os.rename(originalFileName, movedFileName)
+
+def getPropertiesConfigMap(configFileName):
+ ret = {}
+ config = StringIO.StringIO()
+ config.write('[dummysection]\n')
+ config.write(open(configFileName).read())
+ config.seek(0,os.SEEK_SET)
+ fcp = ConfigParser.ConfigParser()
+ fcp.optionxform = str
+ fcp.readfp(config)
+ for k,v in fcp.items('dummysection'):
+ ret[k] = v
+ return ret
+
+def getPropertiesKeyList(configFileName):
+ ret = []
+ config = StringIO.StringIO()
+ config.write('[dummysection]\n')
+ config.write(open(configFileName).read())
+ config.seek(0,os.SEEK_SET)
+ fcp = ConfigParser.ConfigParser()
+ fcp.optionxform = str
+ fcp.readfp(config)
+ for k,v in fcp.items('dummysection'):
+ ret.append(k)
+ return ret
+
+def readFromJCKSFile(jcksFileName,propName):
+ fn = jcksFileName
+ cmd = "java -cp './cred/lib/*' org.apache.ranger.credentialapi.buildks get '" + propName + "' -provider jceks://file" + fn + " 2> /dev/null"
+ pwd = os.popen(cmd).read()
+ pwd = pwd.strip()
+ return pwd
+
+def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName):
+ tree = ET.parse(xmlTemplateFileName)
+ root = tree.getroot()
+ for config in root.iter('property'):
+ name = config.find('name').text
+ if (name in prop.keys()):
+ config.find('value').text = prop[name]
+ else:
+ print "ERROR: key not found: %s" % (name)
+ if isfile(xmlOutputFileName):
+ archiveFile(xmlOutputFileName)
+ tree.write(xmlOutputFileName)
+
+def main():
+ installFileName = join(templateDirectoryName, installPropFileName)
+ installProps = {}
+ rangerprops = {}
+
+ xaSystemPropFileName = join(configDirectory, xaSystemPropFile)
+ xaSysProps = getPropertiesConfigMap(xaSystemPropFileName)
+
+ ldapPropFileName = join(configDirectory, ldapPropFile)
+ xaLdapProps = getPropertiesConfigMap (ldapPropFileName)
+
+ jaasPropFileName = join(rangerJAASDirectoryName, rangerJAASPropFile)
+ unixauthProps = getPropertiesConfigMap (jaasPropFileName)
+
+ webserverConfigFileName = join(configDirectory, webserverConfigFile)
+ webconfig = getPropertiesConfigMap(webserverConfigFileName)
+
+ for k in config2xmlMAP.keys():
+ xmlKey = config2xmlMAP[k]
+ if (k in xaSysProps.keys()):
+ xmlVal = xaSysProps[k]
+ elif (k in xaLdapProps.keys()):
+ xmlVal = xaLdapProps[k]
+ elif (k in unixauthProps.keys()):
+ xmlVal = unixauthProps[k]
+ elif (k in webconfig.keys()):
+ xmlVal = webconfig[k]
+ else:
+ xmlVal = 'Unknown'
+ rangerprops[xmlKey] = xmlVal
+
+ jdbcUrl = xaSysProps['jdbc.url']
+ auditJcksFileName = xaSysProps['auditDB.jdbc.credential.provider.path']
+ jcksFileName = xaSysProps['xaDB.jdbc.credential.provider.path']
+
+
+ tokens = jdbcUrl.split(":")
+ hostTokens = jdbcUrl.split("//")
+ dbTokens = hostTokens[1].split("/")
+
+ libFolderCmd='dirname `readlink -f /usr/bin/ranger-admin`'
+ libFolder = os.popen(libFolderCmd).read().strip() + '/webapp/WEB-INF/lib'
+
+ if (tokens[2] == 'mysql'):
+ installProps['DB_FLAVOR'] = 'MYSQL'
+ installProps['SQL_COMMAND_INVOKER'] = 'mysql'
+ installProps['db_host'] = dbTokens[0]
+ installProps['db_name'] = dbTokens[1]
+ installProps['audit_db_name'] = dbTokens[1]
+ mysqlConnectorJarFileName = [ f for f in listdir(libFolder) if (isfile(join(libFolder,f)) and f.startswith("mysql") and f.endswith(".jar")) ]
+ if (len(mysqlConnectorJarFileName) > 0):
+ if not os.path.exists(tempLibFolder):
+ os.makedirs(tempLibFolder)
+ tempLibFile=join(tempLibFolder,mysqlConnectorJarFileName[0])
+ shutil.copy(join(libFolder,mysqlConnectorJarFileName[0]), tempLibFile)
+ installProps['SQL_CONNECTOR_JAR'] = tempLibFile
+ elif (token[3] == 'odbc'):
+ installProps['DB_FLAVOR'] = 'ORACLE'
+ installProps['SQL_COMMAND_INVOKER'] = 'sqlplus'
+ installProps['db_host'] = dbTokens[0]
+ oraConnectorJarFileName = [ f for f in listdir(libFolder) if (isfile(join(libFolder,f)) and f.startswith("ojdbc") and f.endswith(".jar")) ]
+ if (len(oraConnectorJarFileName) > 0):
+ if not os.path.exists(tempLibFolder):
+ os.makedirs(tempLibFolder)
+ tempLibFile=join(tempLibFolder,oraConnectorJarFileName[0])
+ shutil.copy(join(libFolder,oraConnectorJarFileName[0]), tempLibFile)
+ installProps['SQL_CONNECTOR_JAR'] = tempLibFile
+ #
+ # TODO: for oracle, need to find out as how to get these values
+ #
+ installProps['db_name'] = ''
+ installProps['audit_db_name'] = ''
+ else:
+ print "ERROR: Unable to determine the DB_FLAVOR from url [%]" % (jdbcUrl)
+ sys.exit(1)
+
+ installProps['db_user'] = xaSysProps['jdbc.user']
+ installProps['db_password'] = readFromJCKSFile(jcksFileName, 'policyDB.jdbc.password')
+ installProps['db_root_user'] = 'unknown'
+ installProps['db_root_password'] = 'unknown'
+
+ installProps['audit_db_user']=xaSysProps['auditDB.jdbc.user']
+ installProps['audit_db_password']= readFromJCKSFile(auditJcksFileName, 'auditDB.jdbc.password')
+
+ installProps['policymgr_external_url'] = xaSysProps['xa.webapp.url.root']
+ installProps['policymgr_http_enabled'] = xaSysProps['http.enabled']
+
+ securityContextFileName = join(configDirectory, securityContextFile)
+ tree = ET.parse(securityContextFileName)
+ root = tree.getroot()
+ ns = {'beans' : 'http://www.springframework.org/schema/beans'}
+ if ( len(root.findall(".//*[@id='activeDirectoryAuthenticationProvider']",ns)) > 0):
+ installProps['authentication_method'] = 'AD'
+ installProps['xa_ldap_ad_domain'] = xaLdapProps['xa_ldap_ad_domain']
+ installProps['xa_ldap_ad_url'] = xaLdapProps['xa_ldap_ad_url']
+ elif ( len(root.findall(".//*[@id='ldapAuthProvider']",ns)) > 0 ):
+ installProps['authentication_method'] = 'LDAP'
+ installProps['xa_ldap_url'] = xaLdapProps['xa_ldap_url']
+ installProps['xa_ldap_userDNpattern'] = xaLdapProps['xa_ldap_userDNpattern']
+ installProps['xa_ldap_groupSearchBase'] = xaLdapProps['xa_ldap_groupSearchBase']
+ installProps['xa_ldap_groupSearchFilter'] = xaLdapProps['xa_ldap_groupSearchFilter']
+ installProps['xa_ldap_groupRoleAttribute'] = xaLdapProps['xa_ldap_groupRoleAttribute']
+ elif ( len(root.findall(".//*[@id='jaasAuthProvider']",ns)) > 0 ):
+ installProps['authentication_method'] = 'UNIX'
+ installProps['remoteLoginEnabled'] = unixauthProps['remoteLoginEnabled']
+ installProps['authServiceHostName'] = unixauthProps['authServiceHostName']
+ installProps['authServicePort'] = unixauthProps['authServicePort']
+ else:
+ installProps['authentication_method'] = 'NONE'
+
+ rangerprops['ranger.authentication.method'] = installProps['authentication_method']
+
+ installProps['cred_keystore_filename'] = jcksFileName
+
+ keylist = getPropertiesKeyList(installFileName)
+ defValMap = getPropertiesConfigMap(installFileName)
+
+
+ for wk,wv in webconfig.iteritems():
+ nk = "ranger." + wk
+ nk = nk.replace('.','_')
+ installProps[nk] = wv
+ keylist.append(nk)
+
+ writeToFile(keylist,defValMap,installProps,installPropFileName)
+
+ if (generateXML == 1):
+ writeXMLUsingProperties(join(templateDirectoryName,rangerSiteTemplateXMLFile), rangerprops, rangerSiteXMLFile)
+
+def writeToFile(keyList, defValMap, props, outFileName):
+
+ if (isfile(outFileName)):
+ archiveFile(outFileName)
+
+ outf = open(outFileName, 'w')
+
+ print >> outf, "#"
+ print >> outf, "# -----------------------------------------------------------------------------------"
+ print >> outf, "# This file is generated as part of upgrade script and should be deleted after upgrade"
+ print >> outf, "# Generated at %s " % (strftime("%d/%m/%Y %H:%M:%S", localtime()))
+ print >> outf, "# -----------------------------------------------------------------------------------"
+ print >> outf, "#"
+
+ for key in keyList:
+ if (key in props):
+ print >> outf, "%s=%s" % (key,props[key])
+ val = props[key]
+ else:
+ print >> outf, "# Default value for [%s] is used\n%s=%s\n#---" % (key, key,defValMap[key])
+ val = defValMap[key]
+
+ outf.flush()
+ outf.close()
+
+
+main()
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/admin-web.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml
index 6e5b3c7..f984248 100644
--- a/src/main/assembly/admin-web.xml
+++ b/src/main/assembly/admin-web.xml
@@ -314,10 +314,20 @@
<include>db_setup.py</include>
<include>dba_script.py</include>
<include>restrict_permissions.py</include>
+ <include>upgrade_admin.py</include>
+ <include>upgrade.sh</include>
</includes>
<fileMode>544</fileMode>
</fileSet>
<fileSet>
+ <outputDirectory>/templates-upgrade</outputDirectory>
+ <directory>security-admin/scripts</directory>
+ <includes>
+ <include>install.properties</include>
+ <include>ranger-admin-site-template.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
<outputDirectory>/</outputDirectory>
<directory>security-admin</directory>
<includes>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/hbase-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hbase-agent.xml b/src/main/assembly/hbase-agent.xml
index 62cef98..851ef6f 100644
--- a/src/main/assembly/hbase-agent.xml
+++ b/src/main/assembly/hbase-agent.xml
@@ -125,6 +125,17 @@
<fileMode>755</fileMode>
</file>
<file>
+ <source>agents-common/scripts/upgrade-plugin.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>upgrade-hbase-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>agents-common/scripts/upgrade-plugin.py</source>
+ <outputDirectory>/</outputDirectory>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
<source>agents-common/scripts/enable-agent.sh</source>
<outputDirectory>/</outputDirectory>
<destName>disable-hbase-plugin.sh</destName>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/hdfs-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hdfs-agent.xml b/src/main/assembly/hdfs-agent.xml
index 593aedb..036c2bd 100644
--- a/src/main/assembly/hdfs-agent.xml
+++ b/src/main/assembly/hdfs-agent.xml
@@ -115,6 +115,17 @@
<fileMode>755</fileMode>
</file>
<file>
+ <source>agents-common/scripts/upgrade-plugin.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>upgrade-hdfs-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>agents-common/scripts/upgrade-plugin.py</source>
+ <outputDirectory>/</outputDirectory>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
<source>agents-common/scripts/enable-agent.sh</source>
<outputDirectory>/</outputDirectory>
<destName>disable-hdfs-plugin.sh</destName>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/hive-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hive-agent.xml b/src/main/assembly/hive-agent.xml
index 6137110..c48b1b5 100644
--- a/src/main/assembly/hive-agent.xml
+++ b/src/main/assembly/hive-agent.xml
@@ -125,6 +125,17 @@
<fileMode>755</fileMode>
</file>
<file>
+ <source>agents-common/scripts/upgrade-plugin.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>upgrade-hive-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>agents-common/scripts/upgrade-plugin.py</source>
+ <outputDirectory>/</outputDirectory>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
<source>agents-common/scripts/enable-agent.sh</source>
<outputDirectory>/</outputDirectory>
<destName>disable-hive-plugin.sh</destName>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/knox-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/knox-agent.xml b/src/main/assembly/knox-agent.xml
index 10d3eeb..6552406 100644
--- a/src/main/assembly/knox-agent.xml
+++ b/src/main/assembly/knox-agent.xml
@@ -131,6 +131,17 @@
<fileMode>755</fileMode>
</file>
<file>
+ <source>agents-common/scripts/upgrade-plugin.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>upgrade-knox-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>agents-common/scripts/upgrade-plugin.py</source>
+ <outputDirectory>/</outputDirectory>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
<source>agents-common/scripts/enable-agent.sh</source>
<outputDirectory>/</outputDirectory>
<destName>disable-knox-plugin.sh</destName>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/46b5ecc3/src/main/assembly/storm-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/storm-agent.xml b/src/main/assembly/storm-agent.xml
index 5f32128..998b957 100644
--- a/src/main/assembly/storm-agent.xml
+++ b/src/main/assembly/storm-agent.xml
@@ -139,6 +139,17 @@
<fileMode>755</fileMode>
</file>
<file>
+ <source>agents-common/scripts/upgrade-plugin.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>upgrade-storm-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>agents-common/scripts/upgrade-plugin.py</source>
+ <outputDirectory>/</outputDirectory>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
<source>agents-common/scripts/enable-agent.sh</source>
<outputDirectory>/</outputDirectory>
<destName>disable-storm-plugin.sh</destName>