You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2016/11/08 18:28:43 UTC
lucene-solr:master: SOLR-9038: add snapshotscli.sh and supporting code
Repository: lucene-solr
Updated Branches:
refs/heads/master 750da7c5f -> b02e7a902
SOLR-9038: add snapshotscli.sh and supporting code
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b02e7a90
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b02e7a90
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b02e7a90
Branch: refs/heads/master
Commit: b02e7a902e80c82750f8e8c403b3a9c5e887c43b
Parents: 750da7c
Author: yonik <yo...@apache.org>
Authored: Tue Nov 8 13:28:25 2016 -0500
Committer: yonik <yo...@apache.org>
Committed: Tue Nov 8 13:28:35 2016 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solr/core/snapshots/SolrSnapshotsTool.java | 468 +++++++++++++++++++
.../scripts/cloud-scripts/snapshotscli.sh | 176 +++++++
3 files changed, 646 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b02e7a90/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 77897ef..dc24cf5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -104,6 +104,8 @@ New Features
* SOLR-9682: JSON Facet API: added "param" query type to facet domain filter specification to obtain
filters via query parameters. (yonik)
+* SOLR-9038: Add a command-line tool to manage the snapshots functionality (Hrishikesh Gadre via yonik)
+
Optimizations
----------------------
* SOLR-9704: Facet Module / JSON Facet API: Optimize blockChildren facets that have
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b02e7a90/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java
new file mode 100644
index 0000000..cb1c52c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotsTool.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.snapshots;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.hadoop.fs.Path;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * This class provides utility functions required for Solr snapshots functionality.
+ */
+public class SolrSnapshotsTool implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final DateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
+
+ private static final String CREATE = "create";
+ private static final String DELETE = "delete";
+ private static final String LIST = "list";
+ private static final String DESCRIBE = "describe";
+ private static final String PREPARE_FOR_EXPORT = "prepare-snapshot-export";
+ private static final String EXPORT_SNAPSHOT = "export";
+ private static final String HELP = "help";
+ private static final String COLLECTION = "c";
+ private static final String TEMP_DIR = "t";
+ private static final String DEST_DIR = "d";
+ private static final String SOLR_ZK_ENSEMBLE = "z";
+ private static final String HDFS_PATH_PREFIX = "p";
+ private static final String BACKUP_REPO_NAME = "r";
+ private static final String ASYNC_REQ_ID = "i";
+ private static final List<String> OPTION_HELP_ORDER = Arrays.asList(CREATE, DELETE, LIST, DESCRIBE,
+ PREPARE_FOR_EXPORT, EXPORT_SNAPSHOT, HELP, SOLR_ZK_ENSEMBLE, COLLECTION, DEST_DIR, BACKUP_REPO_NAME,
+ ASYNC_REQ_ID, TEMP_DIR, HDFS_PATH_PREFIX);
+
+ private final CloudSolrClient solrClient;
+
+ public SolrSnapshotsTool(String solrZkEnsemble) {
+ solrClient = (new CloudSolrClient.Builder()).withZkHost(solrZkEnsemble).build();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (solrClient != null) {
+ solrClient.close();
+ }
+ }
+
+ public void createSnapshot(String collectionName, String snapshotName) {
+ CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);
+ CollectionAdminResponse resp;
+ try {
+ resp = createSnap.process(solrClient);
+ Preconditions.checkState(resp.getStatus() == 0, "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());
+ System.out.println("Successfully created snapshot with name " + snapshotName + " for collection " + collectionName);
+
+ } catch (Exception e) {
+ log.error("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName, e);
+ System.out.println("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName
+ +" due to following error : "+e.getLocalizedMessage());
+ }
+ }
+
+ public void deleteSnapshot(String collectionName, String snapshotName) {
+ CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);
+ CollectionAdminResponse resp;
+ try {
+ resp = deleteSnap.process(solrClient);
+ Preconditions.checkState(resp.getStatus() == 0, "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());
+ System.out.println("Successfully deleted snapshot with name " + snapshotName + " for collection " + collectionName);
+
+ } catch (Exception e) {
+ log.error("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName, e);
+ System.out.println("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName
+ +" due to following error : "+e.getLocalizedMessage());
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void listSnapshots(String collectionName) {
+ CollectionAdminRequest.ListSnapshots listSnaps = new CollectionAdminRequest.ListSnapshots(collectionName);
+ CollectionAdminResponse resp;
+ try {
+ resp = listSnaps.process(solrClient);
+ Preconditions.checkState(resp.getStatus() == 0, "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus());
+
+ NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
+ for (int i = 0; i < apiResult.size(); i++) {
+ System.out.println(apiResult.getName(i));
+ }
+
+ } catch (Exception e) {
+ log.error("Failed to list snapshots for collection " + collectionName, e);
+ System.out.println("Failed to list snapshots for collection " + collectionName
+ +" due to following error : "+e.getLocalizedMessage());
+ }
+ }
+
+ public void describeSnapshot(String collectionName, String snapshotName) {
+ try {
+ Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
+ for (CollectionSnapshotMetaData m : snaps) {
+ if (snapshotName.equals(m.getName())) {
+ System.out.println("Name: " + m.getName());
+ System.out.println("Status: " + m.getStatus());
+ System.out.println("Time of creation: " + dateFormat.format(m.getCreationDate()));
+ System.out.println("Total number of cores with snapshot: " + m.getReplicaSnapshots().size());
+ System.out.println("-----------------------------------");
+ for (CoreSnapshotMetaData n : m.getReplicaSnapshots()) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Core [name=");
+ builder.append(n.getCoreName());
+ builder.append(", leader=");
+ builder.append(n.isLeader());
+ builder.append(", generation=");
+ builder.append(n.getGenerationNumber());
+ builder.append(", indexDirPath=");
+ builder.append(n.getIndexDirPath());
+ builder.append("]\n");
+ System.out.println(builder.toString());
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Failed to fetch snapshot details", e);
+ System.out.println("Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());
+ }
+ }
+
+ public Map<String, List<String>> getIndexFilesPathForSnapshot(String collectionName, String snapshotName, Optional<String> pathPrefix)
+ throws SolrServerException, IOException {
+ Map<String, List<String>> result = new HashMap<>();
+
+ Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
+ Optional<CollectionSnapshotMetaData> meta = Optional.empty();
+ for (CollectionSnapshotMetaData m : snaps) {
+ if (snapshotName.equals(m.getName())) {
+ meta = Optional.of(m);
+ }
+ }
+
+ if (!meta.isPresent()) {
+ throw new IllegalArgumentException("The snapshot named " + snapshotName
+ + " is not found for collection " + collectionName);
+ }
+
+ DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
+ for (Slice s : collectionState.getSlices()) {
+ List<CoreSnapshotMetaData> replicaSnaps = meta.get().getReplicaSnapshotsForShard(s.getName());
+ // Prepare a list of *existing* replicas (since one or more replicas could have been deleted after the snapshot creation).
+ List<CoreSnapshotMetaData> availableReplicas = new ArrayList<>();
+ for (CoreSnapshotMetaData m : replicaSnaps) {
+ if (isReplicaAvailable(s, m.getCoreName())) {
+ availableReplicas.add(m);
+ }
+ }
+
+ if (availableReplicas.isEmpty()) {
+ throw new IllegalArgumentException(
+ "The snapshot named " + snapshotName + " not found for shard "
+ + s.getName() + " of collection " + collectionName);
+ }
+
+ // Prefer a leader replica (at the time when the snapshot was created).
+ CoreSnapshotMetaData coreSnap = availableReplicas.get(0);
+ for (CoreSnapshotMetaData m : availableReplicas) {
+ if (m.isLeader()) {
+ coreSnap = m;
+ }
+ }
+
+ String indexDirPath = coreSnap.getIndexDirPath();
+ if (pathPrefix.isPresent()) {
+ // If the path prefix is specified, rebuild the path to the index directory.
+ Path t = new Path(coreSnap.getIndexDirPath());
+ indexDirPath = (new Path(pathPrefix.get(), t.toUri().getPath())).toString();
+ }
+
+ List<String> paths = new ArrayList<>();
+ for (String fileName : coreSnap.getFiles()) {
+ Path p = new Path(indexDirPath, fileName);
+ paths.add(p.toString());
+ }
+
+ result.put(s.getName(), paths);
+ }
+
+ return result;
+ }
+
+ public void buildCopyListings(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix)
+ throws SolrServerException, IOException {
+ Map<String, List<String>> paths = getIndexFilesPathForSnapshot(collectionName, snapshotName, pathPrefix);
+ for (Map.Entry<String,List<String>> entry : paths.entrySet()) {
+ StringBuilder filesBuilder = new StringBuilder();
+ for (String filePath : entry.getValue()) {
+ filesBuilder.append(filePath);
+ filesBuilder.append("\n");
+ }
+
+ String files = filesBuilder.toString().trim();
+ try (Writer w = new OutputStreamWriter(new FileOutputStream(new File(localFsPath, entry.getKey())), StandardCharsets.UTF_8)) {
+ w.write(files);
+ }
+ }
+ }
+
+ public void backupCollectionMetaData(String collectionName, String snapshotName, String backupLoc) throws SolrServerException, IOException {
+ // Backup the collection meta-data
+ CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName);
+ backup.setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
+ backup.setLocation(backupLoc);
+ CollectionAdminResponse resp = backup.process(solrClient);
+ Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus());
+ }
+
+ public void prepareForExport(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix, String destPath) {
+ try {
+ buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix);
+ System.out.println("Successfully prepared copylisting for the snapshot export.");
+ } catch (Exception e) {
+ log.error("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
+ + collectionName, e);
+ System.out.println("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
+ + collectionName + " due to following error : " + e.getLocalizedMessage());
+ System.exit(1);
+ }
+
+ try {
+ backupCollectionMetaData(collectionName, snapshotName, destPath);
+ System.out.println("Successfully backed up collection meta-data");
+ } catch (Exception e) {
+ log.error("Failed to backup collection meta-data for collection " + collectionName, e);
+ System.out.println("Failed to backup collection meta-data for collection " + collectionName
+ + " due to following error : " + e.getLocalizedMessage());
+ System.exit(1);
+ }
+ }
+
+ public void exportSnapshot(String collectionName, String snapshotName, String destPath, Optional<String> backupRepo,
+ Optional<String> asyncReqId) {
+ try {
+ CollectionAdminRequest.Backup backup = new CollectionAdminRequest.Backup(collectionName, snapshotName);
+ backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY);
+ backup.setLocation(destPath);
+ if (backupRepo.isPresent()) {
+ backup.setRepositoryName(backupRepo.get());
+ }
+ if (asyncReqId.isPresent()) {
+ backup.setAsyncId(asyncReqId.get());
+ }
+ CollectionAdminResponse resp = backup.process(solrClient);
+ Preconditions.checkState(resp.getStatus() == 0, "The request failed. The status code is " + resp.getStatus());
+ } catch (Exception e) {
+ log.error("Failed to backup collection meta-data for collection " + collectionName, e);
+ System.out.println("Failed to backup collection meta-data for collection " + collectionName
+ + " due to following error : " + e.getLocalizedMessage());
+ System.exit(1);
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ CommandLineParser parser = new PosixParser();
+ Options options = new Options();
+
+ options.addOption(null, CREATE, true, "This command will create a snapshot with the specified name");
+ options.addOption(null, DELETE, true, "This command will delete a snapshot with the specified name");
+ options.addOption(null, LIST, false, "This command will list all the named snapshots for the specified collection.");
+ options.addOption(null, DESCRIBE, true, "This command will print details for a named snapshot for the specified collection.");
+ options.addOption(null, PREPARE_FOR_EXPORT, true, "This command will prepare copylistings for the specified snapshot."
+ + " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared"
+ + " file-system e.g. HDFS");
+ options.addOption(null, EXPORT_SNAPSHOT, true, "This command will create a backup for the specified snapshot.");
+ options.addOption(null, HELP, false, "This command will print the help message for the snapshots related commands.");
+ options.addOption(TEMP_DIR, true, "This parameter specifies the path of a temporary directory on local filesystem"
+ + " during prepare-snapshot-export command.");
+ options.addOption(DEST_DIR, true, "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related"
+ + " information should be stored.");
+ options.addOption(COLLECTION, true, "This parameter specifies the name of the collection to be used during snapshot operation");
+ options.addOption(SOLR_ZK_ENSEMBLE, true, "This parameter specifies the Solr Zookeeper ensemble address");
+ options.addOption(HDFS_PATH_PREFIX, true, "This parameter specifies the HDFS URI prefix to be used"
+ + " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.");
+ options.addOption(BACKUP_REPO_NAME, true, "This parameter specifies the name of the backup repository to be used"
+ + " during snapshot export preparation");
+ options.addOption(ASYNC_REQ_ID, true, "This parameter specifies the async request identifier to be used"
+ + " during snapshot export preparation");
+
+ CommandLine cmd = null;
+ try {
+ cmd = parser.parse(options, args);
+ } catch (ParseException e) {
+ System.out.println(e.getLocalizedMessage());
+ printHelp(options);
+ System.exit(1);
+ }
+
+ if (cmd.hasOption(CREATE) || cmd.hasOption(DELETE) || cmd.hasOption(LIST) || cmd.hasOption(DESCRIBE)
+ || cmd.hasOption(PREPARE_FOR_EXPORT) || cmd.hasOption(EXPORT_SNAPSHOT)) {
+ try (SolrSnapshotsTool tool = new SolrSnapshotsTool(cmd.getOptionValue(SOLR_ZK_ENSEMBLE))) {
+ if (cmd.hasOption(CREATE)) {
+ String snapshotName = cmd.getOptionValue(CREATE);
+ String collectionName = cmd.getOptionValue(COLLECTION);
+ tool.createSnapshot(collectionName, snapshotName);
+
+ } else if (cmd.hasOption(DELETE)) {
+ String snapshotName = cmd.getOptionValue(DELETE);
+ String collectionName = cmd.getOptionValue(COLLECTION);
+ tool.deleteSnapshot(collectionName, snapshotName);
+
+ } else if (cmd.hasOption(LIST)) {
+ String collectionName = cmd.getOptionValue(COLLECTION);
+ tool.listSnapshots(collectionName);
+
+ } else if (cmd.hasOption(DESCRIBE)) {
+ String snapshotName = cmd.getOptionValue(DESCRIBE);
+ String collectionName = cmd.getOptionValue(COLLECTION);
+ tool.describeSnapshot(collectionName, snapshotName);
+
+ } else if (cmd.hasOption(PREPARE_FOR_EXPORT)) {
+ String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT);
+ String collectionName = cmd.getOptionValue(COLLECTION);
+ String localFsDir = requiredArg(options, cmd, TEMP_DIR);
+ String hdfsOpDir = requiredArg(options, cmd, DEST_DIR);
+ Optional<String> pathPrefix = Optional.ofNullable(cmd.getOptionValue(HDFS_PATH_PREFIX));
+
+ if (pathPrefix.isPresent()) {
+ try {
+ new URI(pathPrefix.get());
+ } catch (URISyntaxException e) {
+ System.out.println(
+ "The specified File system path prefix " + pathPrefix.get()
+ + " is invalid. The error is " + e.getLocalizedMessage());
+ System.exit(1);
+ }
+ }
+ tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir);
+
+ } else if (cmd.hasOption(EXPORT_SNAPSHOT)) {
+ String snapshotName = cmd.getOptionValue(EXPORT_SNAPSHOT);
+ String collectionName = cmd.getOptionValue(COLLECTION);
+ String destDir = requiredArg(options, cmd, DEST_DIR);
+ Optional<String> backupRepo = Optional.ofNullable(cmd.getOptionValue(BACKUP_REPO_NAME));
+ Optional<String> asyncReqId = Optional.ofNullable(cmd.getOptionValue(ASYNC_REQ_ID));
+
+ tool.exportSnapshot(collectionName, snapshotName, destDir, backupRepo, asyncReqId);
+ }
+ }
+ } else if (cmd.hasOption(HELP)) {
+ printHelp(options);
+ } else {
+ System.out.println("Unknown command specified.");
+ printHelp(options);
+ }
+ }
+
+ private static String requiredArg(Options options, CommandLine cmd, String optVal) {
+ if (!cmd.hasOption(optVal)) {
+ System.out.println("Please specify the value for option " + optVal);
+ printHelp(options);
+ System.exit(1);
+ }
+ return cmd.getOptionValue(optVal);
+ }
+
+ private static boolean isReplicaAvailable (Slice s, String coreName) {
+ for (Replica r: s.getReplicas()) {
+ if (coreName.equals(r.getCoreName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(String collectionName)
+ throws SolrServerException, IOException {
+ CollectionAdminRequest.ListSnapshots listSnapshots = new CollectionAdminRequest.ListSnapshots(collectionName);
+ CollectionAdminResponse resp = listSnapshots.process(solrClient);
+
+ Preconditions.checkState(resp.getStatus() == 0);
+
+ NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
+
+ Collection<CollectionSnapshotMetaData> result = new ArrayList<>();
+ for (int i = 0; i < apiResult.size(); i++) {
+ result.add(new CollectionSnapshotMetaData((NamedList<Object>)apiResult.getVal(i)));
+ }
+
+ return result;
+ }
+
+ private static void printHelp(Options options) {
+ StringBuilder helpFooter = new StringBuilder();
+ helpFooter.append("Examples: \n");
+ helpFooter.append("snapshotscli.sh --create snapshot-1 -c books -z localhost:2181 \n");
+ helpFooter.append("snapshotscli.sh --list -c books -z localhost:2181 \n");
+ helpFooter.append("snapshotscli.sh --describe snapshot-1 -c books -z localhost:2181 \n");
+ helpFooter.append("snapshotscli.sh --export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath -i req_0 \n");
+ helpFooter.append("snapshotscli.sh --delete snapshot-1 -c books -z localhost:2181 \n");
+
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.setOptionComparator(new OptionComarator<>());
+ formatter.printHelp("SolrSnapshotsTool", null, options, helpFooter.toString(), false);
+ }
+
+ private static class OptionComarator<T extends Option> implements Comparator<T> {
+
+ public int compare(T o1, T o2) {
+ String s1 = o1.hasLongOpt() ? o1.getLongOpt() : o1.getOpt();
+ String s2 = o2.hasLongOpt() ? o2.getLongOpt() : o2.getOpt();
+ return OPTION_HELP_ORDER.indexOf(s1) - OPTION_HELP_ORDER.indexOf(s2);
+ }
+}
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b02e7a90/solr/server/scripts/cloud-scripts/snapshotscli.sh
----------------------------------------------------------------------
diff --git a/solr/server/scripts/cloud-scripts/snapshotscli.sh b/solr/server/scripts/cloud-scripts/snapshotscli.sh
new file mode 100755
index 0000000..41ae4aa
--- /dev/null
+++ b/solr/server/scripts/cloud-scripts/snapshotscli.sh
@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+
+set -e
+
+run_solr_snapshot_tool() {
+ JVM="java"
+ scriptDir=$(dirname "$0")
+ if [ -n "$LOG4J_PROPS" ]; then
+ log4j_config="file:${LOG4J_PROPS}"
+ else
+ log4j_config="file:${scriptDir}/log4j.properties"
+ fi
+ PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS} -Dlog4j.configuration=${log4j_config} \
+ -classpath "${solrLibPath}" org.apache.solr.core.snapshots.SolrSnapshotsTool "$@" 2> /dev/null
+}
+
+usage() {
+ run_solr_snapshot_tool --help
+}
+
+distcp_warning() {
+ echo "SOLR_USE_DISTCP environment variable is not set. \
+ Do you want to use hadoop distcp tool for exporting Solr collection snapshot ?"
+}
+
+parse_options() {
+ OPTIND=3
+ while getopts ":c:d:s:z:p:r:i:" o ; do
+ case "${o}" in
+ d)
+ destPath=${OPTARG}
+ ;;
+ s)
+ sourcePath=${OPTARG}
+ ;;
+ c)
+ collectionName=${OPTARG}
+ ;;
+ z)
+ solrZkEnsemble=${OPTARG}
+ ;;
+ p)
+ pathPrefix=${OPTARG}
+ ;;
+ r)
+ backupRepoName=${OPTARG}
+ ;;
+ i)
+ aysncReqId=${OPTARG}
+ ;;
+ *)
+ echo "Unknown option ${OPTARG}"
+ usage 1>&2
+ exit 1
+ ;;
+ esac
+ done
+}
+
+prepare_snapshot_export() {
+ #Make sure to cleanup the temporary files.
+ scratch=$(mktemp -d -t solrsnaps.XXXXXXXXXX)
+ function finish {
+ rm -rf "${scratch}"
+ }
+ trap finish EXIT
+
+ if hdfs dfs -test -d "${destPath}" ; then
+ run_solr_snapshot_tool --prepare-snapshot-export "$@" -t "${scratch}"
+
+ hdfs dfs -mkdir -p "${copyListingDirPath}" > /dev/null
+ find "${scratch}" -type f -printf "%f\n" | while read shardId; do
+ echo "Copying the copy-listing for $shardId"
+ hdfs dfs -copyFromLocal "${scratch}/${shardId}" "${copyListingDirPath}" > /dev/null
+ done
+ else
+ echo "Directory ${destPath} does not exist."
+ exit 1
+ fi
+}
+
+copy_snapshot_files() {
+ copylisting_dir_path="$1"
+
+ if hdfs dfs -test -d "${copylisting_dir_path}" ; then
+ for shardId in $(hdfs dfs -stat "%n" "${copylisting_dir_path}/*"); do
+ oPath="${destPath}/${snapshotName}/snapshot.${shardId}"
+ echo "Copying the index files for ${shardId} to ${oPath}"
+ ${distCpCmd} -f " ${copylisting_dir_path}/${shardId}" "${oPath}" > /dev/null
+ done
+ else
+ echo "Directory ${copylisting_dir_path} does not exist."
+ exit 1
+ fi
+}
+
+collectionName=""
+solrZkEnsemble=""
+pathPrefix=""
+destPath=""
+sourcePath=""
+cmd="$1"
+snapshotName="$2"
+copyListingDirPath=""
+distCpCmd="${SOLR_DISTCP_CMD:-hadoop distcp}"
+scriptDir=$(dirname "$0")
+solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../lib/ext/*}"
+
+case "${cmd}" in
+ --create)
+ run_solr_snapshot_tool "$@"
+ ;;
+ --delete)
+ run_solr_snapshot_tool "$@"
+ ;;
+ --list)
+ run_solr_snapshot_tool "$@"
+ ;;
+ --describe)
+ run_solr_snapshot_tool "$@"
+ ;;
+ --prepare-snapshot-export)
+ : "${SOLR_USE_DISTCP:? $(distcp_warning)}"
+
+ parse_options "$@"
+
+ : "${destPath:? Please specify destination directory using -d option}"
+
+ copyListingDirPath="${destPath}/copylistings"
+ prepare_snapshot_export "${@:2}"
+ echo "Done. GoodBye!"
+ ;;
+ --export)
+ if [ -z "${SOLR_USE_DISTCP}" ]; then
+ run_solr_snapshot_tool "$@"
+ echo "Done. GoodBye!"
+ exit 0
+ fi
+
+ parse_options "$@"
+
+ : "${snapshotName:? Please specify the name of the snapshot}"
+ : "${destPath:? Please specify destination directory using -d option}"
+
+ if [ -n "${collectionName}" ] && [ -n "${sourcePath}" ]; then
+ echo "The -c and -s options can not be specified together"
+ exit 1
+ fi
+
+ if [ -z "${collectionName}" ] && [ -z "${sourcePath}" ]; then
+ echo "At least one of options (-c or -s) must be specified"
+ exit 1
+ fi
+
+ if [ -n "${collectionName}" ]; then
+ copyListingDirPath="${destPath}/${snapshotName}/copylistings"
+ prepare_snapshot_export "${@:2}"
+ copy_snapshot_files "${destPath}/${snapshotName}/copylistings"
+ hdfs dfs -rm -r -f -skipTrash "${destPath}/${snapshotName}/copylistings" > /dev/null
+ else
+ copy_snapshot_files "${sourcePath}/copylistings"
+ echo "Copying the collection meta-data to ${destPath}/${snapshotName}"
+ ${distCpCmd} "${sourcePath}/${snapshotName}/*" "${destPath}/${snapshotName}/" > /dev/null
+ fi
+
+ echo "Done. GoodBye!"
+ ;;
+ --help)
+ usage 1>&2
+ ;;
+ *)
+ echo "Unknown command ${cmd}"
+ usage 1>&2
+ exit 1
+esac
+