You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/06/20 00:28:31 UTC
[ambari] branch trunk updated (9ff13b2 -> 690295b)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git.
from 9ff13b2 [AMBARI-24142] Show/Hide config-section not working for Ranger and Ra… (#1577)
new 5273ffc AMBARI-23945. Infra Solr migration - remove 'solr_skip_generate_restore_host_cores' usages.
new f7cafb8 AMBARI-23945. Infra Solr migration - Add --skip-json-dump-files flag
new 690295b AMBARI-23945. Infra Solr migration - Add core filter & skip-json-dump-files to the doc
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ambari-infra/ambari-infra-solr-client/README.md | 9 ++++-
.../src/main/python/migrationHelper.py | 42 ++++++++++++++--------
.../0.1.0/package/scripts/command_commons.py | 2 --
3 files changed, 35 insertions(+), 18 deletions(-)
[ambari] 01/03: AMBARI-23945. Infra Solr migration - remove
'solr_skip_generate_restore_host_cores' usages.
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 5273ffcb3c81334500ef063224681a3ddb3ddd21
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Tue Jun 19 20:20:17 2018 +0200
AMBARI-23945. Infra Solr migration - remove 'solr_skip_generate_restore_host_cores' usages.
---
.../ambari-infra-solr-client/src/main/python/migrationHelper.py | 3 ---
.../AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py | 2 --
2 files changed, 5 deletions(-)
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
index 854bd02..ddcba0c 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
+++ b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
@@ -374,8 +374,6 @@ def fill_parameters(options, config, collection, index_location, hdfs_path=None,
params['solr_hdfs_path'] = hdfs_path
if options.keep_backup:
params['solr_keep_backup'] = True
- if options.skip_generate_restore_host_cores:
- params['solr_skip_generate_restore_host_cores'] = True
return params
def validte_common_options(options, parser, config):
@@ -1425,7 +1423,6 @@ if __name__=="__main__":
parser.add_option("--disable-solr-host-check", dest="disable_solr_host_check", action="store_true", default=False, help="Disable to check solr hosts are good for the collection backups")
parser.add_option("--core-filter", dest="core_filter", default=None, type="string", help="core filter for replica folders")
parser.add_option("--skip-cores", dest="skip_cores", default=None, type="string", help="specific cores to skip (comma separated)")
- parser.add_option("--skip-generate-restore-host-cores", dest="skip_generate_restore_host_cores", default=False, action="store_true", help="Skip the generation of restore_host_cores.json, just read the file itself, can be useful if command failed at some point.")
parser.add_option("--hdfs-base-path", dest="hdfs_base_path", default=None, type="string", help="hdfs base path where the collections are located (e.g.: /user/infrasolr). Use if both atlas and ranger collections are on hdfs.")
parser.add_option("--ranger-hdfs-base-path", dest="ranger_hdfs_base_path", default=None, type="string", help="hdfs base path where the ranger collection is located (e.g.: /user/infra-solr). Use if only ranger collection is on hdfs.")
parser.add_option("--atlas-hdfs-base-path", dest="atlas_hdfs_base_path", default=None, type="string", help="hdfs base path where the atlas collections are located (e.g.: /user/infra-solr). Use if only atlas collections are on hdfs.")
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
index 561a0c4..e9119c9 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
@@ -74,8 +74,6 @@ request_time_interval = int(default("/commandParams/solr_request_time_interval",
check_hosts_default = True if params.security_enabled else False
check_hosts = default("/commandParams/solr_check_hosts", check_hosts_default)
-skip_generate_restore_host_cores = default("/commandParams/solr_skip_generate_restore_host_cores", False)
-
solr_protocol = "https" if params.infra_solr_ssl_enabled else "http"
solr_port = format("{params.infra_solr_port}")
solr_base_url = format("{solr_protocol}://{params.hostname}:{params.infra_solr_port}/solr")
[ambari] 02/03: AMBARI-23945. Infra Solr migration - Add
--skip-json-dump-files flag
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit f7cafb81a29106e2397c720a4cea2ae645089356
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Wed Jun 20 01:49:50 2018 +0200
AMBARI-23945. Infra Solr migration - Add --skip-json-dump-files flag
---
.../src/main/python/migrationHelper.py | 39 +++++++++++++++-------
1 file changed, 27 insertions(+), 12 deletions(-)
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
index ddcba0c..f3e0c44 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
+++ b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
@@ -758,19 +758,33 @@ def copy_znode(options, config, copy_src, copy_dest, copy_from_local=False, copy
logger.debug(str(out))
def list_collections(options, config, output_file):
- solr_cli_command=create_infra_solr_client_command(options, config, '--dump-collections --output {0}'.format(output_file), appendZnode=True)
- logger.debug("Solr cli command: {0}".format(solr_cli_command))
- sys.stdout.write('Dumping collections data to {0} ... '.format(output_file))
- sys.stdout.flush()
- process = Popen(solr_cli_command, stdout=PIPE, stderr=PIPE, shell=True)
- out, err = process.communicate()
- if process.returncode != 0:
- sys.stdout.write(colors.FAIL + 'FAILED\n' + colors.ENDC)
+ dump_json_files_list=[]
+ skip_dump=False
+ if options.skip_json_dump_files:
+ dump_json_files_list=options.skip_json_dump_files.split(',')
+ if dump_json_files_list:
+ for dump_json_file in dump_json_files_list:
+ if output_file.endswith(dump_json_file):
+ skip_dump=True
+ if skip_dump:
+ print 'Skipping collection dump file generation: {0}'.format(output_file)
+ if not os.path.exists(output_file):
+ print "{0}FAIL{1}: Collection dump file '{2}' does not exist.".format(colors.FAIL, colors.ENDC, output_file)
+ sys.exit(1)
+ else:
+ solr_cli_command=create_infra_solr_client_command(options, config, '--dump-collections --output {0}'.format(output_file), appendZnode=True)
+ logger.debug("Solr cli command: {0}".format(solr_cli_command))
+ sys.stdout.write('Dumping collections data to {0} ... '.format(output_file))
sys.stdout.flush()
- raise Exception("{0} command failed: {1}".format(solr_cli_command, str(err)))
- sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC)
- sys.stdout.flush()
- logger.debug(str(out))
+ process = Popen(solr_cli_command, stdout=PIPE, stderr=PIPE, shell=True)
+ out, err = process.communicate()
+ if process.returncode != 0:
+ sys.stdout.write(colors.FAIL + 'FAILED\n' + colors.ENDC)
+ sys.stdout.flush()
+ raise Exception("{0} command failed: {1}".format(solr_cli_command, str(err)))
+ sys.stdout.write(colors.OKGREEN + 'DONE\n' + colors.ENDC)
+ sys.stdout.flush()
+ logger.debug(str(out))
collections_data = get_collections_data(output_file)
return collections_data.keys() if collections_data is not None else []
@@ -1430,6 +1444,7 @@ if __name__=="__main__":
parser.add_option("--batch-interval", dest="batch_interval", type="int", default=60 ,help="batch time interval (seconds) between requests (for restarting INFRA SOLR, default: 60)")
parser.add_option("--batch-fault-tolerance", dest="batch_fault_tolerance", type="int", default=0 ,help="fault tolerance of tasks for batch request (for restarting INFRA SOLR, default: 0)")
parser.add_option("--shared-drive", dest="shared_drive", default=False, action="store_true", help="Use if the backup location is shared between hosts. (override config from config ini file)")
+ parser.add_option("--skip-json-dump-files", dest="skip_json_dump_files", type="string", help="comma separated list of files that won't be download during collection dump (could be useful if it is required to change something in manually in the already downloaded file)")
(options, args) = parser.parse_args()
set_log_level(options.verbose)
[ambari] 03/03: AMBARI-23945. Infra Solr migration - Add core
filter & skip-json-dump-files to the doc
Posted by ol...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 690295bb867e3f357cd438e3d72f75a856032902
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Wed Jun 20 02:25:32 2018 +0200
AMBARI-23945. Infra Solr migration - Add core filter & skip-json-dump-files to the doc
---
ambari-infra/ambari-infra-solr-client/README.md | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/ambari-infra/ambari-infra-solr-client/README.md b/ambari-infra/ambari-infra-solr-client/README.md
index e1bee13..ce86824 100644
--- a/ambari-infra/ambari-infra-solr-client/README.md
+++ b/ambari-infra/ambari-infra-solr-client/README.md
@@ -113,6 +113,7 @@ kerberos_enabled = true
[infra_solr]
protocol = http
hosts = c7402.ambari.apache.org,c7403.ambari.apache.org
+port = 8886
zk_connect_string = c7401.ambari.apache.org:2181
znode = /infra-solr
user = infra-solr
@@ -405,7 +406,7 @@ If the script finished successfully and everything looks green on Ambari UI as w
Migration for `ranger_audits` collection (cores):
```bash
-# by efault, you will mirate to Lucene 6.6.2, if you want to migrate again to Solr 7 (not requred), you can use --version 7.3.1 flag
+# by default, you will mirate to Lucene 6.6.2, if you want to migrate again to Solr 7 (not requred), you can use --version 7.3.1 flag
/usr/lib/ambari-infra-solr-client/migrationHelper.py --ini-file $CONFIG_INI_LOCATION --action migrate -s RANGER
```
@@ -637,6 +638,7 @@ BACKUP_BASE_PATH=/tmp
- `--service-filter` or `-s`: you can filter on services for migration commands (like run against only ATLAS or RANGER), possible values: ATLAS,RANGER,LOGSEARCH
- `--skip-cores`: skip specific cores from migration (can be useful if just one of it failed during restore etc.)
- `--collection` or `-c`: run migration commands on just a specific collection (like: `ranger_adits`, or `old_ranger_audits` for restore)
+- `--core-filter`: can be used only for index migration, that will work as a regex filter on the snapshot core folder e.g.: "mycore" means it will be applied only on "/index/location/mycore_folder" but not on "/index/location/myother_folder"
#### <a id="if-solr-restarted">What to do if Solr instances restarted right after Ambari upgrade but before upgrade Solr instance packages?</a>
@@ -747,6 +749,11 @@ Options:
restarting INFRA SOLR, default: 0)
--shared-drive Use if the backup location is shared between hosts.
(override config from config ini file)
+ --skip-json-dump-files=SKIP_JSON_DUMP_FILES
+ comma separated list of files that won't be download
+ during collection dump (could be useful if it is
+ required to change something in manually in the
+ already downloaded file)
```
#### <a id="migration-config-generator">Solr Migration Config Generator Script</a>