You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/10/02 02:41:47 UTC
[lucene-solr] branch jira/solr-13662 updated: Verification after
every update
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-13662
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/solr-13662 by this push:
new d63c058 Verification after every update
d63c058 is described below
commit d63c058b73ee57fdd4a6e544827e8fbcc9d7c11b
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Wed Oct 2 08:05:03 2019 +0530
Verification after every update
---
solr/core/ivy.xml | 3 +
.../apache/solr/packagemanager/SolrPackage.java | 19 ++++-
.../solr/packagemanager/SolrPackageManager.java | 84 ++++++++++++++++++++--
.../solr/packagemanager/SolrUpdateManager.java | 1 +
.../src/java/org/apache/solr/util/PackageTool.java | 40 +++++++++--
5 files changed, 133 insertions(+), 14 deletions(-)
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index 7805119..3e86702 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -140,6 +140,9 @@
<dependency org="com.jayway.jsonpath" name="json-path" rev="${/com.jayway.jsonpath/json-path}" conf="compile"/>
<dependency org="com.github.zafarkhaja" name="java-semver" rev="0.9.0" conf="compile"/>
+ <dependency org="net.minidev" name="json-smart" rev="2.2.1" conf="compile"/>
+ <dependency org="net.minidev" name="asm" rev="1.0.2" conf="compile"/>
+
<dependency org="org.rrd4j" name="rrd4j" rev="${/org.rrd4j/rrd4j}" conf="compile"/>
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java
index 2908a32..0f6d718 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java
@@ -4,6 +4,7 @@ package org.apache.solr.packagemanager;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -65,14 +66,14 @@ public class SolrPackage implements Serializable, Comparable<SolrPackage> {
public String updateCommand;
@JsonProperty("uninstall-command")
- public String uninstallCommands;
+ public String uninstallCommand;
@JsonProperty("verify-command")
- public String verifyCommand;
+ public Command verifyCommand;
@Override
public String toString() {
- return id + ": {setup: "+setupCommand+", update: "+updateCommand+", uninstall: "+uninstallCommands+", verify: "+verifyCommand+"}";
+ return id + ": {setup: "+setupCommand+", update: "+updateCommand+", uninstall: "+uninstallCommand+", verify: "+verifyCommand+"}";
}
}
@@ -89,5 +90,17 @@ public class SolrPackage implements Serializable, Comparable<SolrPackage> {
this.repositoryId = repositoryId;
}
+ public static class Command {
+ public String path;
+ public String method;
+ public Map payload;
+ public String condition;
+ public String expected;
+
+ @Override
+ public String toString() {
+ return method + " " + path + ", Payload: "+ payload+", Condition: "+condition+", expected: "+expected;
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
index 7128a00..68a88af 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
@@ -15,16 +15,19 @@ import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
+import org.apache.solr.packagemanager.SolrPackage.Command;
import org.apache.solr.packagemanager.SolrPackage.Metadata;
import org.apache.solr.packagemanager.SolrPackage.Plugin;
import org.apache.solr.packagemanager.pf4j.DefaultVersionManager;
import org.apache.solr.packagemanager.pf4j.VersionManager;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.jayway.jsonpath.JsonPath;
public class SolrPackageManager {
@@ -70,22 +73,63 @@ public class SolrPackageManager {
return ret;
}
+ String solrBaseUrl = "http://localhost:8983";
+
public boolean deployInstallPackage(String pluginId, List<String> collections) {
- SolrPackageInstance plugin = getPackage(pluginId);
+ SolrPackageInstance pkg = getPackage(pluginId);
- for (Plugin p: plugin.getPlugins()) {
+ for (Plugin p: pkg.getPlugins()) {
System.out.println(p.setupCommand);
for (String collection: collections) {
System.out.println("Executing " + p.setupCommand + " for collection:" + collection);
postJson("http://localhost:8983/solr/"+collection+"/config", p.setupCommand);
}
}
- return true;
+
+ boolean success = verify(pkg, collections);
+ if (success) {
+ System.out.println("Deployed and verified package: "+pkg.id+", version: "+pkg.version);
+ }
+ return success;
+ }
+
+ //nocommit should this be private?
+ public boolean verify(SolrPackageInstance pkg, List<String> collections) {
+ // verify deployment succeeded?
+ boolean success = true;
+ for (Plugin p: pkg.getPlugins()) {
+ System.out.println(p.verifyCommand);
+ for (String collection: collections) {
+ System.out.println("Executing " + p.verifyCommand + " for collection:" + collection);
+ //postJson("http://localhost:8983/solr/"+collection+"/config", p.setupCommand);
+ Command cmd = p.verifyCommand;
+ String url = solrBaseUrl + resolve(cmd.path, collection, pkg.version, pkg.id);
+
+ if ("GET".equalsIgnoreCase(cmd.method)) {
+ String response = get(url);
+ System.out.println(response);
+ String actualValue = JsonPath.parse(response).read(cmd.condition);
+ String expectedValue = resolve(cmd.expected, collection, pkg.version, pkg.id);
+ System.out.println("Actual: "+actualValue+", expected: "+expectedValue);
+ if (!expectedValue.equals(actualValue)) {
+ System.out.println("Failed to deploy plugin: "+p.id);
+ success = false;
+ }
+ }
+ }
+ }
+ return success;
+ }
+
+ private String resolve(String str, String collection, String packageVersion, String packageName) {
+ return str.replaceAll("\\{collection\\}", collection)
+ .replaceAll("\\{package-version\\}", packageVersion)
+ .replaceAll("\\{package-name\\}", packageName);
}
public boolean deployUpdatePackage(String pluginId, List<String> collections) {
- SolrPackageInstance plugin = getPackage(pluginId);
- for (Plugin p: plugin.getPlugins()) {
+ SolrPackageInstance pkg = getPackage(pluginId);
+ for (Plugin p: pkg.getPlugins()) {
System.out.println(p.updateCommand);
for (String collection: collections) {
@@ -93,10 +137,40 @@ public class SolrPackageManager {
postJson("http://localhost:8983/solr/"+collection+"/config", p.updateCommand);
}
}
+ boolean success = verify(pkg, collections);
+ if (success) {
+ System.out.println("Deployed and verified package: "+pkg.id+", version: "+pkg.version);
+ }
return true;
}
+ private String get(String url) {
+ try (CloseableHttpClient client = HttpClients.createDefault();) {
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setHeader("Content-type", "application/json");
+
+ CloseableHttpResponse response = client.execute(httpGet);
+
+ try {
+ HttpEntity rspEntity = response.getEntity();
+ if (rspEntity != null) {
+ InputStream is = rspEntity.getContent();
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(is, writer, "UTF-8");
+ String results = writer.toString();
+
+ return(results);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
+ return null;
+ }
+
private void postJson(String url, String postBody) {
try (CloseableHttpClient client = HttpClients.createDefault();) {
HttpPost httpPost = new HttpPost(url);
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
index 2db8c4e..a667248 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
@@ -141,6 +141,7 @@ public class SolrUpdateManager {
}
addOrUpdatePackage(op, id, version, sha256, repository, release.sig, metadataSha256, release.metadata);
+
return true;
}
diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java
index ceddba1..02e3f43 100644
--- a/solr/core/src/java/org/apache/solr/util/PackageTool.java
+++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -96,7 +98,7 @@ public class PackageTool extends SolrCLI.ToolBase {
if (cli.getArgList().size()==1) {
update(pluginManager, updateManager);
} else {
- updatePlugin(pluginManager, updateManager, cli.getArgs()[1], cli.getArgList().subList(2, cli.getArgList().size()));
+ updatePackage(pluginManager, updateManager, zkHost, cli.getArgs()[1], cli.getArgList().subList(2, cli.getArgList().size()));
}
break;
default:
@@ -179,16 +181,42 @@ public class PackageTool extends SolrCLI.ToolBase {
}
}
- protected void updatePlugin(SolrPackageManager pluginManager, SolrUpdateManager updateManager, String pluginName, List args) throws PluginException {
+ protected void updatePackage(SolrPackageManager packageManager, SolrUpdateManager updateManager, String zkHost,
+ String packageName, List args) throws PluginException {
if (updateManager.hasUpdates()) {
- String latestVersion = updateManager.getLastPackageRelease(pluginName).version;
- System.out.println("Updating ["+pluginName+"] to version: "+latestVersion);
- updateManager.updatePackage(pluginName, latestVersion);
+ String latestVersion = updateManager.getLastPackageRelease(packageName).version;
+ SolrPackageInstance installedPackage = packageManager.getPackage(packageName);
+ System.out.println("Updating ["+packageName+"] from " + installedPackage.getVersion() + " to version "+latestVersion);
+
+ List<String> collectionsDeployedIn = getDeployedCollections(zkHost, packageManager, installedPackage);
+ System.out.println("Already deployed on collections: "+collectionsDeployedIn);
+ updateManager.updatePackage(packageName, latestVersion);
+
+ SolrPackageInstance updatedPackage = packageManager.getPackage(packageName);
+ System.out.println("Verifying version "+updatedPackage.getVersion()+" on "+collectionsDeployedIn
+ +", result: "+packageManager.verify(updatedPackage, collectionsDeployedIn));
} else {
- System.out.println("Package "+pluginName+" is already up to date.");
+ System.out.println("Package "+packageName+" is already up to date.");
}
}
+ private List<String> getDeployedCollections(String zkHost, SolrPackageManager packageManager, SolrPackageInstance pkg) {
+
+ List<String> allCollections;
+ try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
+ allCollections = zkClient.getChildren("/collections", null, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ System.out.println("Need to verify if these collections have the plugin installed? "+ allCollections);
+ List<String> deployed = new ArrayList<String>();
+ for (String collection: allCollections) {
+ if (packageManager.verify(pkg, Collections.singletonList(collection))) {
+ deployed.add(collection);
+ }
+ }
+ return deployed;
+ }
@SuppressWarnings("static-access")
public Option[] getOptions() {