You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/05/21 21:04:00 UTC
[ambari] branch trunk updated: AMBARI-23822. Add
--skip-generate-restore-host-cores option to migrationHelper.py (only read
the json file, do not generate it)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 50f1169 AMBARI-23822. Add --skip-generate-restore-host-cores option to migrationHelper.py (only read the json file, do not generate it)
50f1169 is described below
commit 50f116950b2bc289d28a53bad3b0c0f4f1070900
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Mon May 21 22:47:28 2018 +0200
AMBARI-23822. Add --skip-generate-restore-host-cores option to migrationHelper.py (only read the json file, do not generate it)
---
ambari-infra/ambari-infra-solr-client/README.md | 3 +++
.../src/main/python/migrationHelper.py | 4 ++++
.../AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py | 4 ++++
.../0.1.0/package/scripts/command_commons.py | 14 +++++++++++++-
4 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/ambari-infra/ambari-infra-solr-client/README.md b/ambari-infra/ambari-infra-solr-client/README.md
index 681b9a1..c2e2c16 100644
--- a/ambari-infra/ambari-infra-solr-client/README.md
+++ b/ambari-infra/ambari-infra-solr-client/README.md
@@ -618,6 +618,9 @@ Options:
core filter for replica folders
--skip-cores=SKIP_CORES
specific cores to skip (comma separated)
+ --skip-generate-restore-host-cores
+ Skip the generation of restore_host_cores.json, just
+ read the file itself, can be useful if command failed at some point.
--shards=SOLR_SHARDS number of shards (required to set properly for
restore)
--solr-hdfs-path=SOLR_HDFS_PATH
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
index 2226346..0328106 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
+++ b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
@@ -152,6 +152,8 @@ def fill_parameters(options):
params['solr_hdfs_path'] = options.solr_hdfs_path
if options.solr_keep_backup:
params['solr_keep_backup'] = True
+ if options.skip_generate_restore_host_cores:
+ params['solr_skip_generate_restore_host_cores'] = True
return params
def validte_common_options(options, parser):
@@ -238,10 +240,12 @@ if __name__=="__main__":
parser.add_option("--disable-solr-host-check", dest="disable_solr_host_check", action="store_true", default=False, help="Disable to check solr hosts are good for the collection backups")
parser.add_option("--core-filter", dest="core_filter", default=None, type="string", help="core filter for replica folders")
parser.add_option("--skip-cores", dest="skip_cores", default=None, type="string", help="specific cores to skip (comma separated)")
+ parser.add_option("--skip-generate-restore-host-cores", dest="skip_generate_restore_host_cores", default=False, action="store_true", help="Skip the generation of restore_host_cores.json, just read the file itself, can be useful if command failed at some point.")
parser.add_option("--shards", dest="solr_shards", type="int", default=0, help="number of shards (required to set properly for restore)")
parser.add_option("--solr-hdfs-path", dest="solr_hdfs_path", type="string", default=None, help="Base path of Solr (where collections are located) if HDFS is used (like /user/infra-solr)")
parser.add_option("--solr-keep-backup", dest="solr_keep_backup", default=False, action="store_true", help="If it is turned on, Snapshot Solr data will not be deleted from the filesystem during restore.")
+
(options, args) = parser.parse_args()
if options.action is None:
parser.print_help()
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
index e22db93..094f782 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
@@ -173,18 +173,21 @@ def restore_collection(env):
recursive_chown=True,
recursive_chmod=True
)
+ command_commons.HdfsResource(None, action="execute")
command_commons.HdfsResource(format("{hdfs_solr_node_folder}/data/tlog"),
type="directory",
action="create_on_execute",
owner=params.infra_solr_user,
mode=0755
)
+ command_commons.HdfsResource(None, action="execute")
command_commons.HdfsResource(format("{hdfs_solr_node_folder}/data/snapshot_metadata"),
type="directory",
action="create_on_execute",
owner=params.infra_solr_user,
mode=0755
)
+ command_commons.HdfsResource(None, action="execute")
if command_commons.solr_keep_backup:
Directory(format("{index_location}/snapshot.{src_core}"),
action="delete",
@@ -239,6 +242,7 @@ def restore_collection(env):
action="delete_on_execute",
owner=params.infra_solr_user
)
+ command_commons.HdfsResource(None, action="execute")
if command_commons.check_hdfs_folder_exists(backup_collection_core_dir):
command_commons.move_hdfs_folder(backup_collection_core_dir, collection_core_dir)
else:
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
index f8613f5..91fff50 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
@@ -74,6 +74,8 @@ request_time_interval = int(default("/commandParams/solr_request_time_interval",
check_hosts_default = True if params.security_enabled else False
check_hosts = default("/commandParams/solr_check_hosts", check_hosts_default)
+skip_generate_restore_host_cores = default("/commandParams/solr_skip_generate_restore_host_cores", False)
+
solr_protocol = "https" if params.infra_solr_ssl_enabled else "http"
solr_port = format("{params.infra_solr_port}")
solr_base_url = format("{solr_protocol}://{params.hostname}:{params.infra_solr_port}/solr")
@@ -316,6 +318,15 @@ def __read_host_cores_from_clusterstate_json(json_zk_state_path, json_host_cores
json.dump(json_content, outfile)
return json_content
+def __read_host_cores_from_file(json_host_cores_path):
+ """
+ Read host cores from file, can be useful if you do not want to regenerate host core data (with that you can generate your own host core pairs for restore)
+ """
+ with open(json_host_cores_path) as json_file:
+ host_cores_json_data = json.load(json_file)
+ return host_cores_json_data
+
+
def get_host_cores_for_collection(backup=True):
"""
Get core details to an object and write them to a file as well. Backup data will be used during restore.
@@ -331,7 +342,8 @@ def get_host_cores_for_collection(backup=True):
json_host_cores_path = format("{json_folder}/restore_host_cores.json")
api_request = create_solr_api_request_command(request_path, output=json_zk_state_path)
Execute(api_request, user=params.infra_solr_user)
- return __read_host_cores_from_clusterstate_json(json_zk_state_path, json_host_cores_path)
+ return __read_host_cores_from_file(json_host_cores_path) if skip_generate_restore_host_cores \
+ else __read_host_cores_from_clusterstate_json(json_zk_state_path, json_host_cores_path)
def read_backup_json():
with open(format("{index_location}/backup_host_cores.json")) as json_file:
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.