You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/03/14 09:02:16 UTC
[ambari] branch trunk updated: AMBARI-23200. Infra Solr - utility
script to run infra-solr custom commands (migrate/backup/restore) (#649)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2cb384f AMBARI-23200. Infra Solr - utility script to run infra-solr custom commands (migrate/backup/restore) (#649)
2cb384f is described below
commit 2cb384f3946870bcd982329ca2f67db0600a3ff5
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Wed Mar 14 10:02:13 2018 +0100
AMBARI-23200. Infra Solr - utility script to run infra-solr custom commands (migrate/backup/restore) (#649)
---
ambari-infra/ambari-infra-solr-client/build.xml | 1 +
.../src/main/python/migrationHelper.py | 235 +++++++++++++++++++++
2 files changed, 236 insertions(+)
diff --git a/ambari-infra/ambari-infra-solr-client/build.xml b/ambari-infra/ambari-infra-solr-client/build.xml
index 725cb62..a5fdfbf 100644
--- a/ambari-infra/ambari-infra-solr-client/build.xml
+++ b/ambari-infra/ambari-infra-solr-client/build.xml
@@ -46,6 +46,7 @@
<fileset file="src/main/resources/solrCloudCli.sh"/>
<fileset file="src/main/resources/solrIndexHelper.sh"/>
<fileset file="src/main/python/solrDataManager.py"/>
+ <fileset file="src/main/python/migrationHelper.py"/>
</copy>
<copy todir="target/package" includeEmptyDirs="no">
<fileset file="src/main/resources/log4j.properties"/>
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
new file mode 100755
index 0000000..b89fd03
--- /dev/null
+++ b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
@@ -0,0 +1,235 @@
+#!/usr/bin/python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys
+import urllib2
+import json
+import base64
+import optparse
+
+HTTP_PROTOCOL = 'http'
+HTTPS_PROTOCOL = 'https'
+
+SOLR_SERVICE_NAME = 'AMBARI_INFRA'
+
+SOLR_COMPONENT_NAME ='INFRA_SOLR'
+
+CLUSTERS_URL = '/api/v1/clusters/{0}'
+
+GET_HOSTS_COMPONENTS_URL = '/services/{0}/components/{1}?fields=host_components'
+
+REQUESTS_API_URL = '/requests'
+
+def api_accessor(host, username, password, protocol, port):
+ def do_request(api_url, request_type, request_body=''):
+ try:
+ url = '{0}://{1}:{2}{3}'.format(protocol, host, port, api_url)
+ print 'Execute {0} {1}'.format(request_type, url)
+ if request_body:
+ print 'Request body: {0}'.format(request_body)
+ admin_auth = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+ request = urllib2.Request(url)
+ request.add_header('Authorization', 'Basic %s' % admin_auth)
+ request.add_header('X-Requested-By', 'ambari')
+ request.add_data(request_body)
+ request.get_method = lambda: request_type
+ response = urllib2.urlopen(request)
+ response_body = response.read()
+ except Exception as exc:
+ raise Exception('Problem with accessing api. Reason: {0}'.format(exc))
+ return response_body
+ return do_request
+
+def format_json(dictionary, tab_level=0):
+ output = ''
+ tab = ' ' * 2 * tab_level
+ for key, value in dictionary.iteritems():
+ output += ',\n{0}"{1}": '.format(tab, key)
+ if isinstance(value, dict):
+ output += '{\n' + format_json(value, tab_level + 1) + tab + '}'
+ else:
+ output += '"{0}"'.format(value)
+ output += '\n'
+ return output[2:]
+
+def read_json(json_file):
+ with open(json_file) as data_file:
+ data = json.load(data_file)
+ return data
+
+def get_json(accessor, url):
+ response = accessor(url, 'GET')
+ print 'GET ' + url + ' response: '
+ print '----------------------------'
+ print response
+ json_resp = json.loads(response)
+ return json_resp
+
+def post_json(accessor, url, request_body):
+ response = accessor(url, 'POST', json.dumps(request_body))
+ print 'POST ' + url + ' response: '
+ print '----------------------------'
+ print response
+ json_resp = json.loads(response)
+ return json_resp
+
+def get_component_hosts(host_components_json):
+ hosts = []
+ if "host_components" in host_components_json and len(host_components_json['host_components']) > 0:
+ for host_component in host_components_json['host_components']:
+ if 'HostRoles' in host_component:
+ hosts.append(host_component['HostRoles']['host_name'])
+ return hosts
+
+def create_command_request(command, parameters, hosts, cluster, context):
+ request = {}
+ request_info = {}
+ request_info["context"] = context
+ request_info["command"] = command
+ request_info["parameters"] = parameters
+
+ operation_level = {}
+ operation_level["level"] = "HOST_COMPONENT"
+ operation_level["cluster_name"] = cluster
+
+ request_info["operation_level"] = operation_level
+ request["RequestInfo"] = request_info
+
+ resource_filter = {}
+ resource_filter["service_name"] = SOLR_SERVICE_NAME
+ resource_filter["component_name"] = SOLR_COMPONENT_NAME
+ resource_filter["hosts"] = ','.join(hosts)
+
+ resource_filters = []
+ resource_filters.append(resource_filter)
+ request["Requests/resource_filters"] = resource_filters
+ return request
+
+def fill_parameters(options):
+ params = {}
+ if options.collection:
+ params['solr_collection'] = options.collection
+ if options.index_location:
+ params['solr_index_location'] = options.index_location
+ if options.backup_name:
+ params['solr_backup_name'] = options.backup_name
+ return params
+
+def validte_common_options(options, parser):
+ if not options.index_location:
+ parser.print_help()
+ print 'index-location option is required'
+ sys.exit(1)
+
+ if not options.collection:
+ parser.print_help()
+ print 'collection option is required'
+ sys.exit(1)
+
+def get_solr_hosts(options, accessor):
+ if options.solr_hosts:
+ component_hosts = options.solr_hosts.split(",")
+ else:
+ host_components_json = get_json(accessor, CLUSTERS_URL.format(options.cluster) + GET_HOSTS_COMPONENTS_URL.format(SOLR_SERVICE_NAME, SOLR_COMPONENT_NAME))
+ component_hosts = get_component_hosts(host_components_json)
+ return component_hosts
+
+def restore(options, accessor, parser):
+ """
+ Send restore solr collection custom command request to ambari-server
+ """
+ validte_common_options(options, parser)
+ if not options.backup_name:
+ parser.print_help()
+ print 'backup-name option is required'
+ sys.exit(1)
+ component_hosts = get_solr_hosts(options, accessor)
+ parameters = fill_parameters(options)
+
+ cmd_request = create_command_request("RESTORE", parameters, component_hosts, options.cluster, 'Restore Solr Collection: ' + options.collection)
+ post_json(accessor, CLUSTERS_URL.format(options.cluster) + REQUESTS_API_URL, cmd_request)
+
+def migrate(options, accessor, parser):
+ """
+ Send migrate lucene index custom command request to ambari-server
+ """
+ validte_common_options(options, parser)
+ component_hosts = get_solr_hosts(options, accessor)
+ parameters = fill_parameters(options)
+
+ cmd_request = create_command_request("MIGRATE", parameters, component_hosts, options.cluster, 'Migrating Solr Collection: ' + options.collection)
+ post_json(accessor, CLUSTERS_URL.format(options.cluster) + REQUESTS_API_URL, cmd_request)
+
+def backup(options, accessor, parser):
+ """
+ Send backup solr collection custom command request to ambari-server
+ """
+ validte_common_options(options, parser)
+ if not options.backup_name:
+ parser.print_help()
+ print 'backup-name option is required'
+ sys.exit(1)
+ component_hosts = get_solr_hosts(options, accessor)
+ parameters = fill_parameters(options)
+
+ cmd_request = create_command_request("BACKUP", parameters, component_hosts, options.cluster, 'Backup Solr Collection: ' + options.collection)
+ post_json(accessor, CLUSTERS_URL.format(options.cluster) + REQUESTS_API_URL, cmd_request)
+
+if __name__=="__main__":
+ parser = optparse.OptionParser("usage: %prog [options]")
+ parser.add_option("-H", "--host", dest="host", default="localhost", type="string", help="hostname for ambari server")
+ parser.add_option("-P", "--port", dest="port", default=8080, type="int", help="port number for ambari server")
+ parser.add_option("-c", "--cluster", dest="cluster", type="string", help="name cluster")
+ parser.add_option("-s", "--ssl", dest="ssl", action="store_true", help="use if ambari server using https")
+ parser.add_option("-u", "--username", dest="username", default="admin", type="string", help="username for accessing ambari server")
+ parser.add_option("-p", "--password", dest="password", default="admin", type="string", help="password for accessing ambari server")
+
+ parser.add_option("-a", "--action", dest="action", type="string", help="backup | restore | migrate ")
+ parser.add_option("-f", "--force", dest="force", default=False, action="store_true", help="force index upgrade even if it's the right version")
+ parser.add_option("--index-location", dest="index_location", type="string", help="location of the index backups")
+ parser.add_option("--backup-name", dest="backup_name", type="string", help="backup name of the index")
+ parser.add_option("--collection", dest="collection", type="string", help="solr collection")
+
+ parser.add_option("--version", dest="index_version", type="string", default="6.6.2", help="lucene index version for migration (6.6.2 or 7.2.1)")
+ parser.add_option("--request-tries", dest="request_tries", type="int", help="number of tries for BACKUP/RESTORE status api calls in the request")
+ parser.add_option("--request-time-interval", dest="request_time_interval", type="int", help="time interval between BACKUP/RESTORE status api calls in the request")
+ parser.add_option("--request-async", dest="async", action="store_true", default=False, help="skip BACKUP/RESTORE status api calls from the command")
+ parser.add_option("--shared-fs", dest="shared_fs", action="store_true", default=False, help="shared fs for storing backup (will create index location to <path><hostname>)")
+ parser.add_option("--solr-hosts", dest="solr_hosts", type="string", help="comma separated list of solr hosts")
+ parser.add_option("--disable-solr-host-check", dest="disable_solr_host_check", action="store_true", default=False, help="Disable to check solr hosts are good for the collection backups")
+ parser.add_option("--core-filter", dest="core_filter", default=None, type="string", help="core filter for replica folders")
+
+ (options, args) = parser.parse_args()
+
+ protocol = 'https' if options.ssl else 'http'
+
+ accessor = api_accessor(options.host, options.username, options.password, protocol, options.port)
+
+ print 'Inputs: ' + str(options)
+ if options.action.lower() == 'backup':
+ backup(options, accessor, parser)
+ elif options.action.lower() == 'restore':
+ pass
+ elif options.action.lower() == 'migrate':
+ pass
+ else:
+ parser.print_help()
+ print 'action option is wrong or missing'
+
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.