You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/10/28 22:01:47 UTC
[lucene-solr] 02/05: Latest auto-update functionality
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-13662-updated
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 11f1e9fa0902b7b46f47b0aa979cbe013af9e556
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Mon Oct 28 21:40:56 2019 +0530
Latest auto-update functionality
---
.../solr/packagemanager/SolrPackageManager.java | 55 +++++++++++++++-------
.../src/java/org/apache/solr/util/PackageTool.java | 38 +++++++++++++--
.../apache/solr/cloud/PackageManagerCLITest.java | 39 +++++++++++++--
3 files changed, 106 insertions(+), 26 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
index e351a68..27fb010 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
@@ -98,23 +98,36 @@ public class SolrPackageManager implements Closeable {
return ret;
}
- public boolean deployInstallPackage(String packageName, String version, List<String> collections, String overrides[]) {
+ Map<String, String> getPackageParams(String packageName, String collection) {
+ try {
+ return (Map<String, String>)((Map)((Map)((Map)new ObjectMapper().readValue
+ (get(solrBaseUrl + "/api/collections/"+collection+"/config/params/packages"), Map.class).get("response")).get("params")).get("packages")).get(packageName);
+ } catch (IOException e) {
+ throw new PackageManagerException(e);
+ }
+
+ }
+
+ public boolean deployInstallPackage(String packageName, String version, boolean isUpdate, List<String> collections, String overrides[]) {
+ boolean pegToLatest = "latest".equals(version); // User wants to peg this package's version to the latest installed (for auto-update, i.e. no explicit deploy step)
SolrPackageInstance pkg = getPackage(packageName, version);
+ if (version == null) {
+ version = pkg.getVersion();
+ }
for (String collection: collections) {
- Map<String, String> collectionParameterOverrides = new HashMap<String,String>();
+ Map<String, String> collectionParameterOverrides = isUpdate? getPackageParams(packageName, collection): new HashMap<String,String>();
if (overrides != null) {
for (String override: overrides) {
collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]);
}
}
+
+ // Get package params
try {
- // nocommit: it overwrites params of other packages (use set or update)
-
boolean packageParamsExist = ((Map)((Map)new ObjectMapper().readValue(
get(solrBaseUrl + "/api/collections/abc/config/params/packages"), Map.class)
).get("response")).containsKey("params");
-
postJson(solrBaseUrl + "/api/collections/"+collection+"/config/params",
new ObjectMapper().writeValueAsString(
Map.of(packageParamsExist? "update": "set",
@@ -123,20 +136,33 @@ public class SolrPackageManager implements Closeable {
throw new RuntimeException(e);
}
+ // Set the package version in the collection's parameters
+ postJson(solrBaseUrl+"/api/collections/abc/config/params", "{set:{PKG_VERSIONS:{"+packageName+" : '"+(pegToLatest? "$LATEST": version)+"'}}}");
+
+ // If updating, refresh the package version for this to take effect
+ if (isUpdate || pegToLatest) {
+ postJson(solrBaseUrl+"/api/cluster/package", "{\"refresh\" : \""+packageName+"\"}");
+ }
+
+ // Setup/update all the plugins in the package
for (Plugin p: pkg.getPlugins()) {
- System.out.println(p.setupCommand);
+ System.out.println(isUpdate? p.updateCommand: p.setupCommand);
Map<String, String> systemParams = new HashMap<String,String>();
systemParams.put("collection", collection);
systemParams.put("package-name", pkg.id);
systemParams.put("package-version", pkg.version);
- String cmd = resolve(p.setupCommand, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
- System.out.println("Executing " + cmd + " for collection:" + collection);
- postJson(solrBaseUrl + "/solr/"+collection+"/config", cmd);
+ String cmd = resolve(isUpdate? p.updateCommand: p.setupCommand, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
+ if (cmd != null && !"".equals(cmd.trim())) {
+ System.out.println("Executing " + cmd + " for collection:" + collection);
+ postJson(solrBaseUrl + "/solr/"+collection+"/config", cmd);
+ }
}
+
}
+ // Verify that package was successfully deployed
boolean success = verify(pkg, collections);
if (success) {
System.out.println("Deployed and verified package: "+pkg.id+", version: "+pkg.version);
@@ -145,6 +171,7 @@ public class SolrPackageManager implements Closeable {
}
private String resolve(String str, Map<String, String> defaults, Map<String, String> overrides, Map<String, String> systemParams) {
+ if (str == null) return null;
for (String param: defaults.keySet()) {
str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param));
}
@@ -164,13 +191,7 @@ public class SolrPackageManager implements Closeable {
System.out.println(p.verifyCommand);
for (String collection: collections) {
System.out.println("Executing " + p.verifyCommand + " for collection:" + collection);
- Map<String, String> collectionParameterOverrides;
- try {
- collectionParameterOverrides = (Map<String, String>)((Map)((Map)((Map)new ObjectMapper().readValue
- (get(solrBaseUrl + "/api/collections/abc/config/params/packages"), Map.class).get("response")).get("params")).get("packages")).get(pkg.id);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ Map<String, String> collectionParameterOverrides = getPackageParams(pkg.id, collection);
Command cmd = p.verifyCommand;
@@ -217,7 +238,7 @@ public class SolrPackageManager implements Closeable {
return get(url);
}
- private String get(String url) {
+ public static String get(String url) {
try (CloseableHttpClient client = SolrUpdateManager.createTrustAllHttpClientBuilder()) {
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader("Content-type", "application/json");
diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java
index f5491e6..e12af04 100644
--- a/solr/core/src/java/org/apache/solr/util/PackageTool.java
+++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java
@@ -48,6 +48,8 @@ import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
public class PackageTool extends SolrCLI.ToolBase {
@@ -64,7 +66,7 @@ public class PackageTool extends SolrCLI.ToolBase {
@Override
protected void runImpl(CommandLine cli) throws Exception {
// Need a logging free, clean output going through to the user.
- Configurator.setRootLevel(Level.OFF);
+ Configurator.setRootLevel(Level.INFO);
solrUrl = cli.getOptionValues("solrUrl")[cli.getOptionValues("solrUrl").length-1];
String solrBaseUrl = solrUrl.replaceAll("\\/solr$", ""); // strip out ending "/solr"
@@ -103,7 +105,10 @@ public class PackageTool extends SolrCLI.ToolBase {
String colls[] = cli.getOptionValues("collections");
String params[] = cli.getOptionValues("param");
System.out.println("coll: "+Arrays.toString(colls)+", params: "+Arrays.toString(params));
- deploy(cli.getArgList().get(1).toString(), colls, params);
+ String packageName = cli.getArgList().get(1).toString().split(":")[0];
+ String version = cli.getArgList().get(1).toString().contains(":")?
+ cli.getArgList().get(1).toString().split(":")[1]: null;
+ deploy(packageName, version, cli.hasOption("update"), colls, params);
break;
case "redeploy":
redeploy(cli.getArgList().subList(1, cli.getArgList().size()));
@@ -175,10 +180,10 @@ public class PackageTool extends SolrCLI.ToolBase {
updateManager.installPackage(args.get(0).toString(), args.get(1).toString());
System.out.println(args.get(0).toString() + " installed.");
}
- protected void deploy(String packageName,
+ protected void deploy(String packageName, String version, boolean isUpdate,
String collections[], String parameters[]) throws PackageManagerException {
- System.out.println(packageManager.deployInstallPackage(packageName.split(":")[0], packageName.split(":").length==2? packageName.split(":")[1]: "latest",
+ System.out.println(packageManager.deployInstallPackage(packageName, version, isUpdate,
Arrays.asList(collections), parameters));
}
@@ -234,8 +239,19 @@ public class PackageTool extends SolrCLI.ToolBase {
System.out.println("Need to verify if these collections have the plugin installed? "+ allCollections);
List<String> deployed = new ArrayList<String>();
for (String collection: allCollections) {
- if (packageManager.verify(pkg, Collections.singletonList(collection))) {
+ // Check package version installed
+ // http://localhost:8983/api/collections/abc/config/params/PKG_VERSIONS?omitHeader=true
+ String paramsJson = SolrPackageManager.get("http://localhost:8983/api/collections/"+collection+"/config/params/PKG_VERSIONS?omitHeader=true");
+ String version = null;
+ try {
+ version = JsonPath.parse(paramsJson).read("$['response'].['params'].['PKG_VERSIONS'].['"+pkg.id+"'])");
+ } catch (PathNotFoundException ex) {
+ // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
+ }
+ if ("$LATEST".equals(version) && packageManager.verify(pkg, Collections.singletonList(collection))) {
deployed.add(collection);
+ } else {
+ System.out.println("Skipping collection: "+collection+", version: "+version);
}
}
return deployed;
@@ -266,6 +282,18 @@ public class PackageTool extends SolrCLI.ToolBase {
.withLongOpt("param")
.create("p"),
+ OptionBuilder
+ .isRequired(false)
+ .withDescription("Solr URL scheme: http or https, defaults to http if not specified")
+ .withLongOpt("update")
+ .create("u"),
+
+ OptionBuilder
+ .isRequired(false)
+ .withDescription("Solr URL scheme: http or https, defaults to http if not specified")
+ .withLongOpt("auto-update")
+ .create(),
+
};
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
index 99286df..b918d2b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud;
+import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -24,9 +25,13 @@ import org.apache.solr.util.PackageTool;
import org.apache.solr.util.SolrCLI;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PackageManagerCLITest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("enable.packages", "true");
@@ -65,16 +70,42 @@ public class PackageManagerCLITest extends SolrCloudTestCase {
assertEquals("tool should have returned 0 for success ", 0, res);
CollectionAdminRequest
- .createCollection("abc", "conf1", 2, 1)
+ .createCollection("abc", "conf1", 1, 1)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
+ CollectionAdminRequest
+ .createCollection("def", "conf1", 1, 1)
+ .setMaxShardsPerNode(100)
+ .process(cluster.getSolrClient());
+
res = run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=/mypath2"});
assertEquals("tool should have returned 0 for success ", 0, res);
- res = run(tool, new String[] {"-solrUrl", solrUrl, "update", "question-answer"});
- assertEquals("tool should have returned 0 for success ", 0, res);
-
+ // Should we test the "auto-update to latest" functionality or the default explicit deploy functionality
+ boolean autoUpdateToLatest = random().nextBoolean();
+
+ if (autoUpdateToLatest) {
+ log.info("Testing auto-update to latest installed");
+ // This command pegs the version to the latest available
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer:latest", "-collections", "abc"});
+ assertEquals("tool should have returned 0 for success ", 0, res);
+
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "update", "question-answer"});
+ assertEquals("tool should have returned 0 for success ", 0, res);
+ } else {
+ log.info("Testing explicit deployment to a different/newer version");
+
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "update", "question-answer"});
+ assertEquals("tool should have returned 0 for success ", 0, res);
+
+ if (random().nextBoolean()) {
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=/mypath2"});
+ } else {
+ res = run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "question-answer", "-collections", "abc"});
+ }
+ assertEquals("tool should have returned 0 for success ", 0, res);
+ }
}
private int run(PackageTool tool, String[] args) throws Exception {