You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/10/17 20:22:24 UTC
[lucene-solr] branch jira/solr-13662 updated: Adding a test and SSL
support
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-13662
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/solr-13662 by this push:
new 70848c2 Adding a test and SSL support
70848c2 is described below
commit 70848c21861f74c6a184c188d7524e11e951fa7b
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Fri Oct 18 01:51:56 2019 +0530
Adding a test and SSL support
---
.../solr/packagemanager/SolrPackageManager.java | 76 ++-
.../solr/packagemanager/SolrUpdateManager.java | 61 +-
.../solr/packagemanager/pf4j/FileDownloader.java | 4 +-
...Exception.java => PackageManagerException.java} | 14 +-
.../packagemanager/pf4j/SimpleFileDownloader.java | 16 +-
.../solr/packagemanager/pf4j/VerifyException.java | 2 +-
.../src/java/org/apache/solr/util/PackageTool.java | 150 +++--
.../apache/solr/cloud/PackageManagerCLITest.java | 729 +--------------------
8 files changed, 201 insertions(+), 851 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
index bc8cd99..164b818 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
@@ -1,11 +1,11 @@
package org.apache.solr.packagemanager;
+import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.MalformedURLException;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -19,59 +19,76 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.packagemanager.SolrPackage.Command;
import org.apache.solr.packagemanager.SolrPackage.Metadata;
import org.apache.solr.packagemanager.SolrPackage.Plugin;
import org.apache.solr.packagemanager.pf4j.DefaultVersionManager;
+import org.apache.solr.packagemanager.pf4j.PackageManagerException;
+import org.apache.zookeeper.KeeperException;
+import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.JsonPath;
-public class SolrPackageManager {
+public class SolrPackageManager implements Closeable {
final DefaultVersionManager versionManager;
final String solrBaseUrl;
- public SolrPackageManager(File repo, String solrBaseUrl) {
+ final SolrZkClient zkClient;
+ public SolrPackageManager(File repo, String solrBaseUrl, String zkHost) {
versionManager = new DefaultVersionManager();
this.solrBaseUrl = solrBaseUrl;
+ this.zkClient = new SolrZkClient(zkHost, 30000);
+ System.out.println("Done initializing a zkClient instance...");
}
Map<String, SolrPackageInstance> packages = null;
Metadata fetchMetadata(String blobSha256) throws MalformedURLException, IOException {
- String metadataJson =
- IOUtils.toString(new URL(solrBaseUrl + "/api/node/blob"+"/"+blobSha256).openStream(), "UTF-8");
+ String metadataJson = getStringFromStream(solrBaseUrl + "/api/node/blob"+"/"+blobSha256);
System.out.println("Fetched metadata blob: "+metadataJson);
Metadata metadata = new ObjectMapper().readValue(metadataJson, Metadata.class);
System.out.println("Now metadata: "+metadata);
return metadata;
}
- public List<SolrPackageInstance> getPackages() {
+ public List<SolrPackageInstance> getPackages() throws PackageManagerException {
System.out.println("Getting packages from clusterprops...");
List<SolrPackageInstance> ret = new ArrayList<SolrPackageInstance>();
packages = new HashMap<String, SolrPackageInstance>();
try {
- String clusterPropsZnode = IOUtils.toString(new URL(solrBaseUrl + "/solr/admin/zookeeper?detail=true&path=/clusterprops.json&wt=json").openStream(), "UTF-8");
+ /*String clusterPropsZnode = IOUtils.toString(new URL(solrBaseUrl + "/solr/admin/zookeeper?detail=true&path=/clusterprops.json&wt=json").openStream(), "UTF-8");
String clusterPropsJson = ((Map)new ObjectMapper().readValue(clusterPropsZnode, Map.class).get("znode")).get("data").toString();
- Map packagesJson = (Map)new ObjectMapper().readValue(clusterPropsJson, Map.class).get("packages");
-
- System.out.println("clusterprops are: "+clusterPropsJson);
- for (Object packageName: packagesJson.keySet()) {
- Map pkg = (Map)packagesJson.get(packageName);
- Metadata metadata = fetchMetadata(pkg.get("metadata").toString());
- List<Plugin> solrplugins = metadata.plugins;
- SolrPackageInstance pkgInstance = new SolrPackageInstance(pkg.get("name").toString(), null,
- pkg.get("version").toString(), solrplugins, metadata.parameterDefaults);
- packages.put(packageName.toString(), pkgInstance);
- ret.add(pkgInstance);
+ Map packagesJson = (Map)new ObjectMapper().readValue(clusterPropsJson, Map.class).get("packages");*/
+
+ String clusterPropsJson = null;
+ Map packagesJson = null;
+
+ if (zkClient.exists("/clusterprops.json", true) == true) {
+ clusterPropsJson = new String(zkClient.getData("/clusterprops.json", null, null, true), "UTF-8");
+ System.out.println("clusterprops are: "+clusterPropsJson);
+ packagesJson = (Map)new ObjectMapper().
+ configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true).readValue(clusterPropsJson, Map.class).get("packages");
}
- } catch (IOException e) {
+
+ if (packagesJson != null) {
+ for (Object packageName: packagesJson.keySet()) {
+ Map pkg = (Map)packagesJson.get(packageName);
+ Metadata metadata = fetchMetadata(pkg.get("metadata").toString());
+ List<Plugin> solrplugins = metadata.plugins;
+ SolrPackageInstance pkgInstance = new SolrPackageInstance(pkg.get("name").toString(), null,
+ pkg.get("version").toString(), solrplugins, metadata.parameterDefaults);
+ packages.put(packageName.toString(), pkgInstance);
+ ret.add(pkgInstance);
+ }
+ }
+ } catch (IOException | KeeperException | InterruptedException e) {
e.printStackTrace();
if (packages == null) packages = Collections.emptyMap(); // nocommit can't happen
+ throw new PackageManagerException(e);
}
return ret;
}
@@ -191,9 +208,12 @@ public class SolrPackageManager {
return true;
}
+ String getStringFromStream(String url) {
+ return get(url);
+ }
private String get(String url) {
- try (CloseableHttpClient client = HttpClients.createDefault();) {
+ try (CloseableHttpClient client = SolrUpdateManager.createTrustAllHttpClientBuilder()) {
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader("Content-type", "application/json");
@@ -212,7 +232,7 @@ public class SolrPackageManager {
} catch (IOException e) {
e.printStackTrace();
}
- } catch (IOException e1) {
+ } catch (Exception e1) {
throw new RuntimeException(e1);
}
return null;
@@ -220,7 +240,7 @@ public class SolrPackageManager {
private void postJson(String url, String postBody) {
System.out.println("Posting to "+url+": "+postBody);
- try (CloseableHttpClient client = HttpClients.createDefault();) {
+ try (CloseableHttpClient client = SolrUpdateManager.createTrustAllHttpClientBuilder();) {
HttpPost httpPost = new HttpPost(url);
StringEntity entity = new StringEntity(postBody);
httpPost.setEntity(entity);
@@ -240,8 +260,9 @@ public class SolrPackageManager {
}
} catch (IOException e) {
e.printStackTrace();
+ throw new RuntimeException(e);
}
- } catch (IOException e1) {
+ } catch (Exception e1) {
throw new RuntimeException(e1);
}
}
@@ -250,4 +271,11 @@ public class SolrPackageManager {
getPackages();
return packages.get(pluginId);
}
+
+ @Override
+ public void close() throws IOException {
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
index 5d845bb..6d676c1 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
@@ -19,6 +19,9 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.FileBody;
@@ -30,7 +33,7 @@ import org.apache.solr.packagemanager.pf4j.CompoundVerifier;
import org.apache.solr.packagemanager.pf4j.DefaultVersionManager;
import org.apache.solr.packagemanager.pf4j.FileDownloader;
import org.apache.solr.packagemanager.pf4j.FileVerifier;
-import org.apache.solr.packagemanager.pf4j.PluginException;
+import org.apache.solr.packagemanager.pf4j.PackageManagerException;
import org.apache.solr.packagemanager.pf4j.SimpleFileDownloader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,11 +80,11 @@ public class SolrUpdateManager {
}
- public synchronized boolean installPackage(String id, String version) throws PluginException {
+ public synchronized boolean installPackage(String id, String version) throws PackageManagerException {
return updateOrInstallPackage(Operation.INSTALL, id, version);
}
- public synchronized boolean updatePackage(String id, String version) throws PluginException {
+ public synchronized boolean updatePackage(String id, String version) throws PackageManagerException {
return updateOrInstallPackage(Operation.UPDATE, id, version);
}
@@ -107,12 +110,12 @@ public class SolrUpdateManager {
return repositories;
}
- private boolean updateOrInstallPackage(Operation op, String id, String version) throws PluginException {
+ private boolean updateOrInstallPackage(Operation op, String id, String version) throws PackageManagerException {
Path downloaded = downloadPackage(id, version);
SolrPackageInstance existingPlugin = packageManager.getPackage(id);
if (existingPlugin != null && version.equals(existingPlugin.getVersion())) {
- throw new PluginException("Plugin already installed.");
+ throw new PackageManagerException("Plugin already installed.");
}
SolrPackageRelease release = null;
@@ -130,7 +133,7 @@ public class SolrUpdateManager {
}
if (release == null) {
- throw new PluginException("Couldn't find the release..");
+ throw new PackageManagerException("Couldn't find the release..");
}
String sha256 = uploadToBlobHandler(downloaded);
@@ -138,7 +141,7 @@ public class SolrUpdateManager {
try {
metadataSha256 = uploadToBlobHandler(new ObjectMapper().writeValueAsString(release.metadata));
} catch (IOException e) {
- throw new PluginException(e);
+ throw new PackageManagerException(e);
}
addOrUpdatePackage(op, id, version, sha256, repository, release.sig, metadataSha256, release.metadata);
@@ -169,16 +172,14 @@ public class SolrUpdateManager {
+ "}}";
System.out.println("Posting package: "+json);
- try (CloseableHttpClient client = HttpClients.createDefault();) {
+ try (CloseableHttpClient client = createTrustAllHttpClientBuilder()) {
HttpPost httpPost = new HttpPost(solrBaseUrl + "/api/cluster/package");
StringEntity entity = new StringEntity(json);
httpPost.setEntity(entity);
httpPost.setHeader("Accept", "application/json");
httpPost.setHeader("Content-type", "application/json");
- CloseableHttpResponse response = client.execute(httpPost);
-
- try {
+ try (CloseableHttpResponse response = client.execute(httpPost)) {
HttpEntity rspEntity = response.getEntity();
if (rspEntity != null) {
InputStream is = rspEntity.getContent();
@@ -190,18 +191,24 @@ public class SolrUpdateManager {
} catch (IOException e) {
e.printStackTrace();
}
-
-
- } catch (IOException e) {
+ } catch (Exception e) {
e.printStackTrace();
}
return true;
}
- private String uploadToBlobHandler(Path downloaded) {
+ public static CloseableHttpClient createTrustAllHttpClientBuilder() throws Exception {
+ SSLContextBuilder builder = new SSLContextBuilder();
+ builder.loadTrustMaterial(null, (chain, authType) -> true);
+ SSLConnectionSocketFactory sslsf = new
+ SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
+ return HttpClients.custom().setSSLSocketFactory(sslsf).build();
+ }
+
+ private String uploadToBlobHandler(Path downloaded) throws PackageManagerException {
String url = solrBaseUrl + "/api/cluster/blob";
File file = downloaded.toFile();
- try (CloseableHttpClient client = HttpClients.createDefault();) {
+ try (CloseableHttpClient client = createTrustAllHttpClientBuilder()) { //HttpClients.createDefault();) {
HttpPost post = new HttpPost(url);
MultipartEntity entity = new MultipartEntity();
@@ -224,17 +231,17 @@ public class SolrUpdateManager {
}
} catch (IOException e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ throw e;
}
- } catch (IOException e1) {
+ } catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
+ throw new PackageManagerException(e1);
}
return null;
}
- private String uploadToBlobHandler(String json) throws IOException {
-
+ private String uploadToBlobHandler(String json) throws IOException, PackageManagerException {
System.out.println("Trying to upload the blob: "+json);
FileUtils.writeStringToFile(new File("tmp-metadata"), json);
return uploadToBlobHandler(new File("tmp-metadata").toPath());
@@ -247,9 +254,9 @@ public class SolrUpdateManager {
* @param id of plugin
* @param version of plugin or null to download latest
* @return Path to file which will reside in a temporary folder in the system default temp area
- * @throws PluginException if download failed
+ * @throws PackageManagerException if download failed
*/
- protected Path downloadPackage(String id, String version) throws PluginException {
+ protected Path downloadPackage(String id, String version) throws PackageManagerException {
try {
SolrPackageRelease release = findReleaseForPlugin(id, version);
Path downloaded = getFileDownloader(id).downloadFile(new URL(release.url));
@@ -257,7 +264,7 @@ public class SolrUpdateManager {
//nocommit verify this download
return downloaded;
} catch (IOException e) {
- throw new PluginException(e, "Error during download of plugin {}", id);
+ throw new PackageManagerException(e, "Error during download of plugin {}", id);
}
}
@@ -300,13 +307,13 @@ public class SolrUpdateManager {
* @param id of plugin
* @param version of plugin or null to locate latest version
* @return PluginRelease for downloading
- * @throws PluginException if id or version does not exist
+ * @throws PackageManagerException if id or version does not exist
*/
- protected SolrPackageRelease findReleaseForPlugin(String id, String version) throws PluginException {
+ protected SolrPackageRelease findReleaseForPlugin(String id, String version) throws PackageManagerException {
SolrPackage pluginInfo = getPackagesMap().get(id);
if (pluginInfo == null) {
log.info("Plugin with id {} does not exist in any repository", id);
- throw new PluginException("Plugin with id {} not found in any repository", id);
+ throw new PackageManagerException("Plugin with id {} not found in any repository", id);
}
if (version == null) {
@@ -319,7 +326,7 @@ public class SolrUpdateManager {
}
}
- throw new PluginException("Plugin {} with version @{} does not exist in the repository", id, version);
+ throw new PackageManagerException("Plugin {} with version @{} does not exist in the repository", id, version);
}
/**
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/FileDownloader.java b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/FileDownloader.java
index c77aeb8..32f6fe4 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/FileDownloader.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/FileDownloader.java
@@ -33,7 +33,7 @@ public interface FileDownloader {
* @param fileUrl the URL representing the file to download
* @return Path of downloaded file, typically in a temporary folder
* @throws IOException if there was an IO problem during download
- * @throws PluginException in case of other problems, such as unsupported protocol
+ * @throws PackageManagerException in case of other problems, such as unsupported protocol
*/
- Path downloadFile(URL fileUrl) throws PluginException, IOException;
+ Path downloadFile(URL fileUrl) throws PackageManagerException, IOException;
}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/PluginException.java b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/PackageManagerException.java
similarity index 69%
rename from solr/core/src/java/org/apache/solr/packagemanager/pf4j/PluginException.java
rename to solr/core/src/java/org/apache/solr/packagemanager/pf4j/PackageManagerException.java
index 974bc82..76fb761 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/PluginException.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/PackageManagerException.java
@@ -21,29 +21,29 @@ package org.apache.solr.packagemanager.pf4j;
*
* @author Decebal Suiu
*/
-public class PluginException extends Exception {
+public class PackageManagerException extends RuntimeException {
- public PluginException() {
+ public PackageManagerException() {
super();
}
- public PluginException(String message) {
+ public PackageManagerException(String message) {
super(message);
}
- public PluginException(Throwable cause) {
+ public PackageManagerException(Throwable cause) {
super(cause);
}
- public PluginException(String message, Throwable cause) {
+ public PackageManagerException(String message, Throwable cause) {
super(message, cause);
}
- public PluginException(Throwable cause, String message, Object... args) {
+ public PackageManagerException(Throwable cause, String message, Object... args) {
super(StringUtils.format(message, args), cause);
}
- public PluginException(String message, Object... args) {
+ public PackageManagerException(String message, Object... args) {
super(StringUtils.format(message, args));
}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/SimpleFileDownloader.java b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/SimpleFileDownloader.java
index c0a4c93..5b7c3ae 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/SimpleFileDownloader.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/SimpleFileDownloader.java
@@ -48,9 +48,9 @@ public class SimpleFileDownloader implements FileDownloader {
* @param fileUrl the URI representing the file to download
* @return the path of downloaded/copied file
* @throws IOException in case of network or IO problems
- * @throws PluginException in case of other problems
+ * @throws PackageManagerException in case of other problems
*/
- public Path downloadFile(URL fileUrl) throws PluginException, IOException {
+ public Path downloadFile(URL fileUrl) throws PackageManagerException, IOException {
switch (fileUrl.getProtocol()) {
case "http":
case "https":
@@ -59,7 +59,7 @@ public class SimpleFileDownloader implements FileDownloader {
case "file":
return copyLocalFile(fileUrl);
default:
- throw new PluginException("URL protocol {} not supported", fileUrl.getProtocol());
+ throw new PackageManagerException("URL protocol {} not supported", fileUrl.getProtocol());
}
}
@@ -69,9 +69,9 @@ public class SimpleFileDownloader implements FileDownloader {
* @param fileUrl source file
* @return path of target file
* @throws IOException if problems during copy
- * @throws PluginException in case of other problems
+ * @throws PackageManagerException in case of other problems
*/
- protected Path copyLocalFile(URL fileUrl) throws IOException, PluginException {
+ protected Path copyLocalFile(URL fileUrl) throws IOException, PackageManagerException {
Path destination = Files.createTempDirectory("pf4j-update-downloader");
destination.toFile().deleteOnExit();
@@ -84,7 +84,7 @@ public class SimpleFileDownloader implements FileDownloader {
return toFile;
} catch (URISyntaxException e) {
- throw new PluginException("Something wrong with given URL", e);
+ throw new PackageManagerException("Something wrong with given URL", e);
}
}
@@ -94,9 +94,9 @@ public class SimpleFileDownloader implements FileDownloader {
* @param fileUrl source file
* @return path of downloaded file
* @throws IOException if IO problems
- * @throws PluginException if validation fails or any other problems
+ * @throws PackageManagerException if validation fails or any other problems
*/
- protected Path downloadFileHttp(URL fileUrl) throws IOException, PluginException {
+ protected Path downloadFileHttp(URL fileUrl) throws IOException, PackageManagerException {
Path destination = Files.createTempDirectory("pf4j-update-downloader");
destination.toFile().deleteOnExit();
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/VerifyException.java b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/VerifyException.java
index 649f44f..e893963 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/pf4j/VerifyException.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/pf4j/VerifyException.java
@@ -18,7 +18,7 @@ package org.apache.solr.packagemanager.pf4j;
/**
* Marker exception for plugin verification failure
*/
-public class VerifyException extends PluginException {
+public class VerifyException extends PackageManagerException {
public VerifyException(String message) {
super(message);
}
diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java
index 77cb55c..fa8220b 100644
--- a/solr/core/src/java/org/apache/solr/util/PackageTool.java
+++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java
@@ -42,7 +42,7 @@ import org.apache.solr.packagemanager.SolrPackageInstance;
import org.apache.solr.packagemanager.SolrPackageManager;
import org.apache.solr.packagemanager.SolrPackageRepository;
import org.apache.solr.packagemanager.SolrUpdateManager;
-import org.apache.solr.packagemanager.pf4j.PluginException;
+import org.apache.solr.packagemanager.pf4j.PackageManagerException;
import org.apache.solr.util.SolrCLI.StatusTool;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -60,7 +60,7 @@ public class PackageTool extends SolrCLI.ToolBase {
public SolrPackageManager packageManager;
public SolrUpdateManager updateManager;
-
+
@Override
protected void runImpl(CommandLine cli) throws Exception {
// Need a logging free, clean output going through to the user.
@@ -71,76 +71,82 @@ public class PackageTool extends SolrCLI.ToolBase {
System.out.println("solr url: "+solrUrl+", solr base url: "+solrBaseUrl);
String zkHost = getZkHost(cli);
-
+
System.out.println("ZK: "+zkHost);
String cmd = cli.getArgs()[0];
- if (cmd != null) {
- packageManager = new SolrPackageManager(new File("./plugins"), solrBaseUrl);
- updateManager = new SolrUpdateManager(packageManager,
- getRepositoriesJson(new SolrZkClient(zkHost, 30000)), solrBaseUrl);
-
- switch (cmd) {
- case "add-repo":
- addRepo(zkHost, cli.getArgs()[1], cli.getArgs()[2]);
- break;
- case "list":
- list(cli.getArgList().subList(1, cli.getArgList().size()));
- break;
- case "list-available":
- try {
- available(cli.getArgList().subList(1, cli.getArgList().size()));
- } catch (PluginException ex) {
- ex.printStackTrace();
- }
- break;
- case "install":
- install(cli.getArgList().subList(1, cli.getArgList().size()));
- break;
- case "deploy":
- String colls[] = cli.getOptionValues("collections");
- String params[] = cli.getOptionValues("param");
- System.out.println("coll: "+Arrays.toString(colls)+", params: "+Arrays.toString(params));
- deploy(cli.getArgList().get(1).toString(), colls, params);
- break;
- case "redeploy":
- redeploy(cli.getArgList().subList(1, cli.getArgList().size()));
- break;
- case "update":
- if (cli.getArgList().size()==1) {
- update();
- } else {
- updatePackage(zkHost, cli.getArgs()[1], cli.getArgList().subList(2, cli.getArgList().size()));
- }
- break;
- default:
- throw new RuntimeException("Unrecognized command: "+cmd);
- };
+ try (SolrZkClient zkclient = new SolrZkClient(zkHost, 30000)) {
+ if (cmd != null) {
+ packageManager = new SolrPackageManager(new File("./plugins"), solrBaseUrl, zkHost);
+ try {
+ updateManager = new SolrUpdateManager(packageManager,
+ getRepositoriesJson(zkclient), solrBaseUrl);
+
+ switch (cmd) {
+ case "add-repo":
+ addRepo(zkHost, cli.getArgs()[1], cli.getArgs()[2]);
+ break;
+ case "list":
+ list(cli.getArgList().subList(1, cli.getArgList().size()));
+ break;
+ case "list-available":
+ try {
+ available(cli.getArgList().subList(1, cli.getArgList().size()));
+ } catch (PackageManagerException ex) {
+ ex.printStackTrace();
+ }
+ break;
+ case "install":
+ install(cli.getArgList().subList(1, cli.getArgList().size()));
+ break;
+ case "deploy":
+ String colls[] = cli.getOptionValues("collections");
+ String params[] = cli.getOptionValues("param");
+ System.out.println("coll: "+Arrays.toString(colls)+", params: "+Arrays.toString(params));
+ deploy(cli.getArgList().get(1).toString(), colls, params);
+ break;
+ case "redeploy":
+ redeploy(cli.getArgList().subList(1, cli.getArgList().size()));
+ break;
+ case "update":
+ if (cli.getArgList().size()==1) {
+ update();
+ } else {
+ updatePackage(zkHost, cli.getArgs()[1], cli.getArgList().subList(2, cli.getArgList().size()));
+ }
+ break;
+ default:
+ throw new RuntimeException("Unrecognized command: "+cmd);
+ };
+ } finally {
+ packageManager.close();
+ }
+ }
}
System.out.println("khatam"); // nocommit
}
protected void addRepo(String zkHost, String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException {
- SolrZkClient zkClient = new SolrZkClient(zkHost, 30000);
+ try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
+ String existingRepositoriesJson = getRepositoriesJson(zkClient);
+ System.out.println(existingRepositoriesJson);
+
+ List repos = new ObjectMapper().readValue(existingRepositoriesJson, List.class);
+ repos.add(new SolrPackageRepository(name, uri));
+ if (zkClient.exists("/repositories.json", true) == false) {
+ zkClient.create("/repositories.json", new ObjectMapper().writeValueAsString(repos).getBytes(), CreateMode.PERSISTENT, true);
+ } else {
+ zkClient.setData("/repositories.json", new ObjectMapper().writeValueAsString(repos).getBytes(), true);
+ }
- String existingRepositoriesJson = getRepositoriesJson(zkClient);
- System.out.println(existingRepositoriesJson);
+ if (zkClient.exists("/keys", true)==false) zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true);
+ if (zkClient.exists("/keys/exe", true)==false) zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true);
+ if (zkClient.exists("/keys/exe/"+"pub_key.der", true)==false) zkClient.create("/keys/exe/"+"pub_key.der", new byte[0], CreateMode.PERSISTENT, true);
+ zkClient.setData("/keys/exe/"+"pub_key.der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true);
- List repos = new ObjectMapper().readValue(existingRepositoriesJson, List.class);
- repos.add(new SolrPackageRepository(name, uri));
- if (zkClient.exists("/repositories.json", true) == false) {
- zkClient.create("/repositories.json", new ObjectMapper().writeValueAsString(repos).getBytes(), CreateMode.PERSISTENT, true);
- } else {
- zkClient.setData("/repositories.json", new ObjectMapper().writeValueAsString(repos).getBytes(), true);
+ System.out.println("Added repository: "+name);
+ System.out.println(getRepositoriesJson(zkClient));
}
-
- if (zkClient.exists("/keys", true)==false) zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true);
- if (zkClient.exists("/keys/exe", true)==false) zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true);
- if (zkClient.exists("/keys/exe/"+"pub_key.der", true)==false) zkClient.create("/keys/exe/"+"pub_key.der", new byte[0], CreateMode.PERSISTENT, true);
- zkClient.setData("/keys/exe/"+"pub_key.der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true);
-
- System.out.println("Added repository: "+name);
- System.out.println(getRepositoriesJson(zkClient));
}
protected String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException {
@@ -155,7 +161,7 @@ public class PackageTool extends SolrCLI.ToolBase {
System.out.println(pkg.getPluginId()+" ("+pkg.getVersion()+")");
}
}
- protected void available(List args) throws PluginException {
+ protected void available(List args) throws PackageManagerException {
System.out.println("Available packages:\n-----");
for (SolrPackage i: updateManager.getPackages()) {
SolrPackage plugin = (SolrPackage)i;
@@ -165,20 +171,20 @@ public class PackageTool extends SolrCLI.ToolBase {
}
}
}
- protected void install(List args) throws PluginException {
+ protected void install(List args) throws PackageManagerException {
updateManager.installPackage(args.get(0).toString(), args.get(1).toString());
System.out.println(args.get(0).toString() + " installed.");
}
protected void deploy(String packageName,
- String collections[], String parameters[]) throws PluginException {
+ String collections[], String parameters[]) throws PackageManagerException {
System.out.println(packageManager.deployInstallPackage(packageName, Arrays.asList(collections), parameters));
}
- protected void redeploy(List args) throws PluginException {
+ protected void redeploy(List args) throws PackageManagerException {
System.out.println(packageManager.deployUpdatePackage(args.get(0).toString(), args.subList(1, args.size())));
}
- protected void update() throws PluginException {
+ protected void update() throws PackageManagerException {
if (updateManager.hasUpdates()) {
System.out.println("Available updates:\n-----");
@@ -194,16 +200,16 @@ public class PackageTool extends SolrCLI.ToolBase {
}
}
- protected void updatePackage(String zkHost, String packageName, List args) throws PluginException {
+ protected void updatePackage(String zkHost, String packageName, List args) throws PackageManagerException {
if (updateManager.hasUpdates()) {
String latestVersion = updateManager.getLastPackageRelease(packageName).version;
SolrPackageInstance installedPackage = packageManager.getPackage(packageName);
System.out.println("Updating ["+packageName+"] from " + installedPackage.getVersion() + " to version "+latestVersion);
-
+
List<String> collectionsDeployedIn = getDeployedCollections(zkHost, packageManager, installedPackage);
System.out.println("Already deployed on collections: "+collectionsDeployedIn);
updateManager.updatePackage(packageName, latestVersion);
-
+
SolrPackageInstance updatedPackage = packageManager.getPackage(packageName);
System.out.println("Verifying version "+updatedPackage.getVersion()+" on "+collectionsDeployedIn
+", result: "+packageManager.verify(updatedPackage, collectionsDeployedIn));
@@ -213,7 +219,7 @@ public class PackageTool extends SolrCLI.ToolBase {
}
private List<String> getDeployedCollections(String zkHost, SolrPackageManager packageManager, SolrPackageInstance pkg) {
-
+
List<String> allCollections;
try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
allCollections = zkClient.getChildren("/collections", null, true);
@@ -229,7 +235,7 @@ public class PackageTool extends SolrCLI.ToolBase {
}
return deployed;
}
-
+
@SuppressWarnings("static-access")
public Option[] getOptions() {
return new Option[] {
@@ -239,7 +245,7 @@ public class PackageTool extends SolrCLI.ToolBase {
.isRequired(true)
.withDescription("Address of the Solr Web application, defaults to: "+SolrCLI.DEFAULT_SOLR_URL)
.create("solrUrl"),
-
+
OptionBuilder
.withArgName("COLLECTIONS")
.hasArgs()
diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
index ccd6b1b..e4dd46d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
@@ -17,21 +17,11 @@
package org.apache.solr.cloud;
-import java.io.IOException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.util.PackageTool;
import org.apache.solr.util.SolrCLI;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -39,728 +29,47 @@ public class PackageManagerCLITest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
+ System.setProperty("enable.package", "true");
+ System.setProperty("enable.runtime.lib", "true");
+
configureCluster(1)
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
- zkAddr = cluster.getZkServer().getZkAddress();
- zkClient = new SolrZkClient(zkAddr, 30000);
-
- }
-
- @AfterClass
- public static void closeConn() {
- if (null != zkClient) {
- zkClient.close();
- zkClient = null;
- }
- zkAddr = null;
}
- private static String zkAddr;
- private static SolrZkClient zkClient;
-
@Test
public void testUpconfig() throws Exception {
// Use a full, explicit path for configset.
Path configSet = TEST_PATH().resolve("configsets");
Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "upconfig1", zkAddr);
- // Now do we have that config up on ZK?
- verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig1");
+ AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "upconfig1", cluster.getZkServer().getZkAddress());
// Now just use a name in the configsets directory, do we find it?
configSet = TEST_PATH().resolve("configsets");
- /*String[] args = new String[]{
- "-confname", "upconfig2",
- "-confdir", "cloud-subdirs",
- "-zkHost", zkAddr,
- "-configsetsDir", configSet.toAbsolutePath().toString(),
- };
-
- SolrCLI.ConfigSetUploadTool tool = new SolrCLI.ConfigSetUploadTool();*/
-
- String args[] = {"-solrUrl", cluster.getJettySolrRunner(0).getBaseUrl().toString(), "list"};
PackageTool tool = new PackageTool();
-
- int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
+ String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();
+ int res = run(tool, new String[] {"-solrUrl", solrUrl, "list"});
assertEquals("tool should have returned 0 for success ", 0, res);
- /*// Now do we have that config up on ZK?
- verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig2");
-
- // do we barf on a bogus path?
- args = new String[]{
- "-confname", "upconfig3",
- "-confdir", "nothinghere",
- "-zkHost", zkAddr,
- "-configsetsDir", configSet.toAbsolutePath().toString(),
- };
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- assertTrue("tool should have returned non-zero for failure ", 0 != res);
-
- String content = new String(zkClient.getData("/configs/upconfig2/schema.xml", null, null, true), StandardCharsets.UTF_8);
- assertTrue("There should be content in the node! ", content.contains("Apache Software Foundation"));*/
-
- }
-
-/* @Test
- public void testDownconfig() throws Exception {
- Path tmp = Paths.get(createTempDir("downConfigNewPlace").toAbsolutePath().toString(), "myconfset");
-
- // First we need a configset on ZK to bring down.
- Path configSet = TEST_PATH().resolve("configsets");
- Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "downconfig1", zkAddr);
- // Now do we have that config up on ZK?
- verifyZkLocalPathsMatch(srcPathCheck, "/configs/downconfig1");
-
- String[] args = new String[]{
- "-confname", "downconfig1",
- "-confdir", tmp.toAbsolutePath().toString(),
- "-zkHost", zkAddr,
- };
-
- SolrCLI.ConfigSetDownloadTool downTool = new SolrCLI.ConfigSetDownloadTool();
- int res = downTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(downTool.getOptions()), args));
- assertEquals("Download should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(Paths.get(tmp.toAbsolutePath().toString(), "conf"), "/configs/downconfig1");
-
- // Insure that empty files don't become directories (SOLR-11198)
-
- Path emptyFile = Paths.get(tmp.toAbsolutePath().toString(), "conf", "stopwords", "emptyfile");
- Files.createFile(emptyFile);
-
- // Now copy it up and back and insure it's still a file in the new place
- AbstractDistribZkTestBase.copyConfigUp(tmp.getParent(), "myconfset", "downconfig2", zkAddr);
- Path tmp2 = createTempDir("downConfigNewPlace2");
- downTool = new SolrCLI.ConfigSetDownloadTool();
- args = new String[]{
- "-confname", "downconfig2",
- "-confdir", tmp2.toAbsolutePath().toString(),
- "-zkHost", zkAddr,
- };
-
- res = downTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(downTool.getOptions()), args));
- assertEquals("Download should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(Paths.get(tmp.toAbsolutePath().toString(), "conf"), "/configs/downconfig2");
- // And insure the empty file is a text file
- Path destEmpty = Paths.get(tmp2.toAbsolutePath().toString(), "conf", "stopwords", "emptyfile");
- assertTrue("Empty files should NOT be copied down as directories", destEmpty.toFile().isFile());
-
- }
-
- @Test
- public void testCp() throws Exception {
- // First get something up on ZK
-
- Path configSet = TEST_PATH().resolve("configsets");
- Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
-
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "cp1", zkAddr);
-
- // Now copy it somewhere else on ZK.
- String[] args = new String[]{
- "-src", "zk:/configs/cp1",
- "-dst", "zk:/cp2",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- SolrCLI.ZkCpTool cpTool = new SolrCLI.ZkCpTool();
-
- int res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy from zk -> zk should have succeeded.", 0, res);
- verifyZnodesMatch("/configs/cp1", "/cp2");
-
-
- // try with zk->local
- Path tmp = createTempDir("tmpNewPlace2");
- args = new String[]{
- "-src", "zk:/configs/cp1",
- "-dst", "file:" + tmp.toAbsolutePath().toString(),
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(tmp, "/configs/cp1");
-
-
- // try with zk->local no file: prefix
- tmp = createTempDir("tmpNewPlace3");
- args = new String[]{
- "-src", "zk:/configs/cp1",
- "-dst", tmp.toAbsolutePath().toString(),
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(tmp, "/configs/cp1");
-
-
- // try with local->zk
- args = new String[]{
- "-src", srcPathCheck.toAbsolutePath().toString(),
- "-dst", "zk:/cp3",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(srcPathCheck, "/cp3");
-
- // try with local->zk, file: specified
- args = new String[]{
- "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
- "-dst", "zk:/cp4",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(srcPathCheck, "/cp4");
-
- // try with recurse not specified
- args = new String[]{
- "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
- "-dst", "zk:/cp5Fail",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertTrue("Copy should NOT have succeeded, recurse not specified.", 0 != res);
-
- // try with recurse = false
- args = new String[]{
- "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
- "-dst", "zk:/cp6Fail",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertTrue("Copy should NOT have succeeded, recurse set to false.", 0 != res);
-
-
- // NOTE: really can't test copying to '.' because the test framework doesn't allow altering the source tree
- // and at least IntelliJ's CWD is in the source tree.
-
- // copy to local ending in separator
- //src and cp3 and cp4 are valid
- String localSlash = tmp.normalize() + File.separator +"cpToLocal" + File.separator;
- args = new String[]{
- "-src", "zk:/cp3/schema.xml",
- "-dst", localSlash,
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should nave created intermediate directory locally.", 0, res);
- assertTrue("File should have been copied to a directory successfully", Files.exists(Paths.get(localSlash, "schema.xml")));
-
- // copy to ZK ending in '/'.
- //src and cp3 are valid
- args = new String[]{
- "-src", "file:" + srcPathCheck.normalize().toAbsolutePath().toString() + File.separator + "solrconfig.xml",
- "-dst", "zk:/powerup/",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy up to intermediate file should have succeeded.", 0, res);
- assertTrue("Should have created an intermediate node on ZK", zkClient.exists("/powerup/solrconfig.xml", true));
-
- // copy individual file up
- //src and cp3 are valid
- args = new String[]{
- "-src", "file:" + srcPathCheck.normalize().toAbsolutePath().toString() + File.separator + "solrconfig.xml",
- "-dst", "zk:/copyUpFile.xml",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy up to named file should have succeeded.", 0, res);
- assertTrue("Should NOT have created an intermediate node on ZK", zkClient.exists("/copyUpFile.xml", true));
-
- // copy individual file down
- //src and cp3 are valid
-
- String localNamed = tmp.normalize().toString() + File.separator + "localnamed" + File.separator + "renamed.txt";
- args = new String[]{
- "-src", "zk:/cp4/solrconfig.xml",
- "-dst", "file:" + localNamed,
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy to local named file should have succeeded.", 0, res);
- Path locPath = Paths.get(localNamed);
- assertTrue("Should have found file: " + localNamed, Files.exists(locPath));
- assertTrue("Should be an individual file", Files.isRegularFile(locPath));
- assertTrue("File should have some data", Files.size(locPath) > 100);
- boolean foundApache = false;
- for (String line : Files.readAllLines(locPath, Charset.forName("UTF-8"))) {
- if (line.contains("Apache Software Foundation")) {
- foundApache = true;
- break;
- }
- }
- assertTrue("Should have found Apache Software Foundation in the file! ", foundApache);
-
-
- // Test copy from somwehere in ZK to the root of ZK.
- args = new String[]{
- "-src", "zk:/cp4/solrconfig.xml",
- "-dst", "zk:/",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy from somewhere in ZK to ZK root should have succeeded.", 0, res);
- assertTrue("Should have found znode /solrconfig.xml: ", zkClient.exists("/solrconfig.xml", true));
-
- // Check that the form path/ works for copying files up. Should append the last bit of the source path to the dst
- args = new String[]{
- "-src", "file:" + srcPathCheck.toAbsolutePath().toString(),
- "-dst", "zk:/cp7/",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
- verifyZkLocalPathsMatch(srcPathCheck, "/cp7/" + srcPathCheck.getFileName().toString());
-
- // Check for an intermediate ZNODE having content. You know cp7/stopwords is a parent node.
- tmp = createTempDir("dirdata");
- Path file = Paths.get(tmp.toAbsolutePath().toString(), "zknode.data");
- List<String> lines = new ArrayList<>();
- lines.add("{Some Arbitrary Data}");
- Files.write(file, lines, Charset.forName("UTF-8"));
- // First, just copy the data up the cp7 since it's a directory.
- args = new String[]{
- "-src", "file:" + file.toAbsolutePath().toString(),
- "-dst", "zk:/cp7/conf/stopwords/",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- String content = new String(zkClient.getData("/cp7/conf/stopwords", null, null, true), StandardCharsets.UTF_8);
- assertTrue("There should be content in the node! ", content.contains("{Some Arbitrary Data}"));
-
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- tmp = createTempDir("cp8");
- args = new String[]{
- "-src", "zk:/cp7",
- "-dst", "file:" + tmp.toAbsolutePath().toString(),
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- // Next, copy cp7 down and verify that zknode.data exists for cp7
- Path zData = Paths.get(tmp.toAbsolutePath().toString(), "conf/stopwords/zknode.data");
- assertTrue("znode.data should have been copied down", zData.toFile().exists());
-
- // Finally, copy up to cp8 and verify that the data is up there.
- args = new String[]{
- "-src", "file:" + tmp.toAbsolutePath().toString(),
- "-dst", "zk:/cp9",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- content = new String(zkClient.getData("/cp9/conf/stopwords", null, null, true), StandardCharsets.UTF_8);
- assertTrue("There should be content in the node! ", content.contains("{Some Arbitrary Data}"));
-
- // Copy an individual empty file up and back down and insure it's still a file
- Path emptyFile = Paths.get(tmp.toAbsolutePath().toString(), "conf", "stopwords", "emptyfile");
- Files.createFile(emptyFile);
-
- args = new String[]{
- "-src", "file:" + emptyFile.toAbsolutePath().toString(),
- "-dst", "zk:/cp7/conf/stopwords/emptyfile",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- Path tmp2 = createTempDir("cp9");
- Path emptyDest = Paths.get(tmp2.toAbsolutePath().toString(), "emptyfile");
- args = new String[]{
- "-src", "zk:/cp7/conf/stopwords/emptyfile",
- "-dst", "file:" + emptyDest.toAbsolutePath().toString(),
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- assertTrue("Empty files should NOT be copied down as directories", emptyDest.toFile().isFile());
-
- // Now with recursive copy
-
- args = new String[]{
- "-src", "file:" + emptyFile.getParent().getParent().toString(),
- "-dst", "zk:/cp10",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- // Now copy it all back and make sure empty file is still a file when recursively copying.
- tmp2 = createTempDir("cp10");
- args = new String[]{
- "-src", "zk:/cp10",
- "-dst", "file:" + tmp2.toAbsolutePath().toString(),
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
- res = cpTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args));
- assertEquals("Copy should have succeeded.", 0, res);
-
- Path locEmpty = Paths.get(tmp2.toAbsolutePath().toString(), "stopwords", "emptyfile");
- assertTrue("Empty files should NOT be copied down as directories", locEmpty.toFile().isFile());
- }
-
- @Test
- public void testMv() throws Exception {
-
- // First get something up on ZK
-
- Path configSet = TEST_PATH().resolve("configsets");
- Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
-
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "mv1", zkAddr);
-
- // Now move it somewhere else.
- String[] args = new String[]{
- "-src", "zk:/configs/mv1",
- "-dst", "zk:/mv2",
- "-zkHost", zkAddr,
- };
-
- SolrCLI.ZkMvTool mvTool = new SolrCLI.ZkMvTool();
-
- int res = mvTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args));
- assertEquals("Move should have succeeded.", 0, res);
-
- // Now does the moved directory match the original on disk?
- verifyZkLocalPathsMatch(srcPathCheck, "/mv2");
- // And are we sure the old path is gone?
- assertFalse("/configs/mv1 Znode should not be there: ", zkClient.exists("/configs/mv1", true));
-
- // Files are in mv2
- // Now fail if we specify "file:". Everything should still be in /mv2
- args = new String[]{
- "-src", "file:" + File.separator + "mv2",
- "-dst", "/mv3",
- "-zkHost", zkAddr,
- };
-
- // Still in mv2
- res = mvTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args));
- assertTrue("Move should NOT have succeeded with file: specified.", 0 != res);
-
- // Let's move it to yet another place with no zk: prefix.
- args = new String[]{
- "-src", "/mv2",
- "-dst", "/mv4",
- "-zkHost", zkAddr,
- };
-
- res = mvTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args));
- assertEquals("Move should have succeeded.", 0, res);
-
- assertFalse("Znode /mv3 really should be gone", zkClient.exists("/mv3", true));
-
- // Now does the moved directory match the original on disk?
- verifyZkLocalPathsMatch(srcPathCheck, "/mv4");
-
- args = new String[]{
- "-src", "/mv4/solrconfig.xml",
- "-dst", "/testmvsingle/solrconfig.xml",
- "-zkHost", zkAddr,
- };
-
- res = mvTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args));
- assertEquals("Move should have succeeded.", 0, res);
- assertTrue("Should be able to move a single file", zkClient.exists("/testmvsingle/solrconfig.xml", true));
-
- zkClient.makePath("/parentNode", true);
-
- // what happens if the destination ends with a slash?
- args = new String[]{
- "-src", "/mv4/schema.xml",
- "-dst", "/parentnode/",
- "-zkHost", zkAddr,
- };
-
- res = mvTool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args));
- assertEquals("Move should have succeeded.", 0, res);
- assertTrue("Should be able to move a single file to a parent znode", zkClient.exists("/parentnode/schema.xml", true));
- String content = new String(zkClient.getData("/parentnode/schema.xml", null, null, true), StandardCharsets.UTF_8);
- assertTrue("There should be content in the node! ", content.contains("Apache Software Foundation"));
- }
-
- @Test
- public void testLs() throws Exception {
-
- Path configSet = TEST_PATH().resolve("configsets");
-
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "lister", zkAddr);
-
- // Should only find a single level.
- String[] args = new String[]{
- "-path", "/configs",
- "-zkHost", zkAddr,
- };
-
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream ps = new PrintStream(baos, false, StandardCharsets.UTF_8.name());
- SolrCLI.ZkLsTool tool = new SolrCLI.ZkLsTool(ps);
-
-
- int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- String content = new String(baos.toByteArray(), StandardCharsets.UTF_8);
-
- assertEquals("List should have succeeded", res, 0);
- assertTrue("Return should contain the conf directory", content.contains("lister"));
- assertFalse("Return should NOT contain a child node", content.contains("solrconfig.xml"));
-
-
- // simple ls recurse=false
- args = new String[]{
- "-path", "/configs",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- content = new String(baos.toByteArray(), StandardCharsets.UTF_8);
-
- assertEquals("List should have succeeded", res, 0);
- assertTrue("Return should contain the conf directory", content.contains("lister"));
- assertFalse("Return should NOT contain a child node", content.contains("solrconfig.xml"));
-
- // recurse=true
- args = new String[]{
- "-path", "/configs",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- content = new String(baos.toByteArray(), StandardCharsets.UTF_8);
-
- assertEquals("List should have succeeded", res, 0);
- assertTrue("Return should contain the conf directory", content.contains("lister"));
- assertTrue("Return should contain a child node", content.contains("solrconfig.xml"));
-
- // Saw a case where going from root foo'd, so test it.
- args = new String[]{
- "-path", "/",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- content = new String(baos.toByteArray(), StandardCharsets.UTF_8);
-
- assertEquals("List should have succeeded", res, 0);
- assertTrue("Return should contain the conf directory", content.contains("lister"));
- assertTrue("Return should contain a child node", content.contains("solrconfig.xml"));
-
- args = new String[]{
- "-path", "/",
- "-zkHost", zkAddr,
- };
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "add-repo", "fullstory", "http://localhost:8081"});
+ assertEquals("tool should have returned 0 for success ", 0, res);
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- content = new String(baos.toByteArray(), StandardCharsets.UTF_8);
- assertEquals("List should have succeeded", res, 0);
- assertFalse("Return should not contain /zookeeper", content.contains("/zookeeper"));
-
- // Saw a case where ending in slash foo'd, so test it.
- args = new String[]{
- "-path", "/configs/",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- content = new String(baos.toByteArray(), StandardCharsets.UTF_8);
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer", "1.0.0"});
+ assertEquals("tool should have returned 0 for success ", 0, res);
- assertEquals("List should have succeeded", res, 0);
- assertTrue("Return should contain the conf directory", content.contains("lister"));
- assertTrue("Return should contain a child node", content.contains("solrconfig.xml"));
+ CollectionAdminRequest
+ .createCollection("abc", "conf1", 2, 1)
+ .setMaxShardsPerNode(100)
+ .process(cluster.getSolrClient());
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=/mypath2"});
+ assertEquals("tool should have returned 0 for success ", 0, res);
}
- @Test
- public void testRm() throws Exception {
-
- Path configSet = TEST_PATH().resolve("configsets");
- Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf");
-
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm1", zkAddr);
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm2", zkAddr);
-
- // Should fail if recurse not set.
- String[] args = new String[]{
- "-path", "/configs/rm1",
- "-zkHost", zkAddr,
- };
-
- SolrCLI.ZkRmTool tool = new SolrCLI.ZkRmTool();
-
+ private int run(PackageTool tool, String[] args) throws Exception {
int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
-
- assertTrue("Should have failed to remove node with children unless -recurse is set to true", res != 0);
-
- // Are we sure all the znodes are still there?
- verifyZkLocalPathsMatch(srcPathCheck, "/configs/rm1");
-
- args = new String[]{
- "-path", "zk:/configs/rm1",
- "-recurse", "false",
- "-zkHost", zkAddr,
- };
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
-
- assertTrue("Should have failed to remove node with children if -recurse is set to false", res != 0);
-
- args = new String[]{
- "-path", "/configs/rm1",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- assertEquals("Should have removed node /configs/rm1", res, 0);
- assertFalse("Znode /configs/toremove really should be gone", zkClient.exists("/configs/rm1", true));
-
- // Check that zk prefix also works.
- args = new String[]{
- "-path", "zk:/configs/rm2",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
-
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- assertEquals("Should have removed node /configs/rm2", res, 0);
- assertFalse("Znode /configs/toremove2 really should be gone", zkClient.exists("/configs/rm2", true));
-
- // This should silently just refuse to do anything to the / or /zookeeper
- args = new String[]{
- "-path", "zk:/",
- "-recurse", "true",
- "-zkHost", zkAddr,
- };
-
- AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm3", zkAddr);
- res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
- assertFalse("Should fail when trying to remove /.", res == 0);
- }*/
-
- // Check that all children of fileRoot are children of zkRoot and vice-versa
- private void verifyZkLocalPathsMatch(Path fileRoot, String zkRoot) throws IOException, KeeperException, InterruptedException {
- verifyAllFilesAreZNodes(fileRoot, zkRoot);
- verifyAllZNodesAreFiles(fileRoot, zkRoot);
- }
-
- private static boolean isEphemeral(String zkPath) throws KeeperException, InterruptedException {
- Stat znodeStat = zkClient.exists(zkPath, null, true);
- return znodeStat.getEphemeralOwner() != 0;
- }
-
- void verifyAllZNodesAreFiles(Path fileRoot, String zkRoot) throws KeeperException, InterruptedException {
-
- for (String child : zkClient.getChildren(zkRoot, null, true)) {
- // Skip ephemeral nodes
- if (zkRoot.endsWith("/") == false) zkRoot += "/";
- if (isEphemeral(zkRoot + child)) continue;
-
- Path thisPath = Paths.get(fileRoot.toAbsolutePath().toString(), child);
- assertTrue("Znode " + child + " should have been found on disk at " + fileRoot.toAbsolutePath().toString(),
- Files.exists(thisPath));
- verifyAllZNodesAreFiles(thisPath, zkRoot + child);
- }
- }
-
- void verifyAllFilesAreZNodes(Path fileRoot, String zkRoot) throws IOException {
- Files.walkFileTree(fileRoot, new SimpleFileVisitor<Path>() {
- void checkPathOnZk(Path path) {
- String znode = ZkMaintenanceUtils.createZkNodeName(zkRoot, fileRoot, path);
- try { // It's easier to catch this exception and fail than catch it everywher eles.
- assertTrue("Should have found " + znode + " on Zookeeper", zkClient.exists(znode, true));
- } catch (Exception e) {
- fail("Caught unexpected exception " + e.getMessage() + " Znode we were checking " + znode);
- }
- }
-
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- assertTrue("Path should start at proper place!", file.startsWith(fileRoot));
- checkPathOnZk(file);
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-
- checkPathOnZk(dir);
- return FileVisitResult.CONTINUE;
- }
- });
- }
-
- // Insure that all znodes in first are in second and vice-versa
- private void verifyZnodesMatch(String first, String second) throws KeeperException, InterruptedException {
- verifyFirstZNodesInSecond(first, second);
- verifyFirstZNodesInSecond(second, first);
- }
-
- // Note, no folderol here with Windows path names.
- private void verifyFirstZNodesInSecond(String first, String second) throws KeeperException, InterruptedException {
- for (String node : zkClient.getChildren(first, null, true)) {
- String fNode = first + "/" + node;
- String sNode = second + "/" + node;
- assertTrue("Node " + sNode + " not found. Exists on " + fNode, zkClient.exists(sNode, true));
- verifyFirstZNodesInSecond(fNode, sNode);
- }
+ return res;
}
}