You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/11/04 15:07:07 UTC
[lucene-solr] branch jira/solr-13662-3 created (now 9d9d415)
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a change to branch jira/solr-13662-3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
at 9d9d415 SOLR-13662: Package Manager (CLI), first cut
This branch includes the following new commits:
new 9d9d415 SOLR-13662: Package Manager (CLI), first cut
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene-solr] 01/01: SOLR-13662: Package Manager (CLI), first cut
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-13662-3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 9d9d415e1395d7f955a265da4fc11dcec5407a12
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Mon Nov 4 20:36:36 2019 +0530
SOLR-13662: Package Manager (CLI), first cut
---
solr/bin/solr | 58 ++++
solr/core/ivy.xml | 4 +
.../apache/solr/packagemanager/PackageUtils.java | 128 ++++++++
.../apache/solr/packagemanager/SolrPackage.java | 107 +++++++
.../solr/packagemanager/SolrPackageInstance.java | 42 +++
.../solr/packagemanager/SolrPackageManager.java | 258 +++++++++++++++
.../solr/packagemanager/SolrPackageRepository.java | 130 ++++++++
.../solr/packagemanager/SolrUpdateManager.java | 218 +++++++++++++
.../src/java/org/apache/solr/pkg/PackageAPI.java | 8 +
.../src/java/org/apache/solr/util/PackageTool.java | 350 +++++++++++++++++++++
.../src/java/org/apache/solr/util/SolrCLI.java | 2 +
.../solr/question-answer-repository/manifest.json | 65 ++++
.../solr/question-answer-repository/publickey.der | Bin 0 -> 94 bytes
.../question-answer-request-handler-1.0.jar | Bin 0 -> 5652 bytes
.../question-answer-request-handler-1.1.jar | Bin 0 -> 5745 bytes
.../question-answer-request-handler-1.2.jar | Bin 0 -> 5741 bytes
.../apache/solr/cloud/PackageManagerCLITest.java | 184 +++++++++++
.../solr/client/solrj/request/beans/Package.java | 4 +
18 files changed, 1558 insertions(+)
diff --git a/solr/bin/solr b/solr/bin/solr
index 596242f..cb82610 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -764,6 +764,59 @@ function get_info() {
return $CODE
} # end get_info
+function run_package() {
+ runningSolrUrl=""
+
+ numSolrs=`find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | wc -l | tr -d ' '`
+ if [ "$numSolrs" != "0" ]; then
+ echo -e "\nFound $numSolrs Solr nodes: "
+ while read PIDF
+ do
+ ID=`cat "$PIDF"`
+ port=`jetty_port "$ID"`
+ if [ "$port" != "" ]; then
+ echo -e "\nSolr process $ID running on port $port"
+ #run_tool status -solr "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr"
+ runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr"
+ break
+ CODE=$?
+ echo ""
+ else
+ echo -e "\nSolr process $ID from $PIDF not found."
+ CODE=1
+ fi
+ done < <(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f)
+ else
+ # no pid files but check using ps just to be sure
+ numSolrs=`ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//'`
+ if [ "$numSolrs" != "0" ]; then
+ echo -e "\nFound $numSolrs Solr nodes: "
+ PROCESSES=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | awk '{print $2}' | sort -r)
+ for ID in $PROCESSES
+ do
+ port=`jetty_port "$ID"`
+ if [ "$port" != "" ]; then
+ echo ""
+ echo "Solr process $ID running on port $port"
+ runningSolrUrl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr"
+ break
+ CODE=$?
+ echo ""
+ fi
+ done
+ else
+ echo -e "\nNo Solr nodes are running.\n"
+ exit 1
+ CODE=3
+ fi
+ fi
+
+ echo "Solr Base URL is $runningSolrUrl"
+ echo "Params: $@"
+ run_tool package -solrUrl "$runningSolrUrl" $@
+ #exit $?
+}
+
# tries to gracefully stop Solr using the Jetty
# stop command and if that fails, then uses kill -9
function stop_solr() {
@@ -1359,6 +1412,11 @@ if [[ "$SCRIPT_CMD" == "export" ]]; then
exit $?
fi
+if [[ "$SCRIPT_CMD" == "package" ]]; then
+ run_package $@
+ exit $?
+fi
+
if [[ "$SCRIPT_CMD" == "auth" ]]; then
VERBOSE=""
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index 9fba663..76de7c4 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -138,6 +138,10 @@
<dependency org="com.google.protobuf" name="protobuf-java" rev="${/com.google.protobuf/protobuf-java}" conf="compile"/>
<dependency org="com.jayway.jsonpath" name="json-path" rev="${/com.jayway.jsonpath/json-path}" conf="compile"/>
+ <dependency org="com.github.zafarkhaja" name="java-semver" rev="0.9.0" conf="compile"/>
+ <dependency org="net.minidev" name="json-smart" rev="2.2.1" conf="compile"/>
+ <dependency org="net.minidev" name="asm" rev="1.0.2" conf="compile"/>
+
<dependency org="org.rrd4j" name="rrd4j" rev="${/org.rrd4j/rrd4j}" conf="compile"/>
<!-- JWT Auth plugin -->
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
new file mode 100644
index 0000000..da23068
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.packagemanager;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLContextBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.response.V2Response;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+
+import com.github.zafarkhaja.semver.Version;
+import com.google.common.base.Strings;
+
+public class PackageUtils {
+ static public ByteBuffer getFileContent(File file) throws IOException {
+ ByteBuffer jar;
+ try (FileInputStream fis = new FileInputStream(file)) {
+ byte[] buf = new byte[fis.available()];
+ fis.read(buf);
+ jar = ByteBuffer.wrap(buf);
+ }
+ return jar;
+ }
+
+ public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
+ throws SolrServerException, IOException {
+ String resource = "/api/cluster/files" + name;
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ if (sig != null) {
+ params.add("sig", sig);
+ }
+ V2Response rsp = new V2Request.Builder(resource)
+ .withMethod(SolrRequest.METHOD.PUT)
+ .withPayload(buffer)
+ .forceV2(true)
+ .withMimeType("application/octet-stream")
+ .withParams(params)
+ .build()
+ .process(client);
+ if (!name.equals(rsp.getResponse().get(CommonParams.FILE))) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Mismatch in file uploaded. Uploaded: " +
+ rsp.getResponse().get(CommonParams.FILE)+", Original: "+name);
+ }
+ }
+
+ public static String getStringFromStream(String url) {
+ return get(url);
+ }
+
+ public static String get(String url) {
+ try (CloseableHttpClient client = PackageUtils.createTrustAllHttpClientBuilder()) {
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setHeader("Content-type", "application/json");
+
+ CloseableHttpResponse response = client.execute(httpGet);
+
+ try {
+ HttpEntity rspEntity = response.getEntity();
+ if (rspEntity != null) {
+ InputStream is = rspEntity.getContent();
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(is, writer, "UTF-8");
+ String results = writer.toString();
+
+ return(results);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ } catch (Exception e1) {
+ throw new RuntimeException(e1);
+ }
+ return null;
+ }
+
+ public static CloseableHttpClient createTrustAllHttpClientBuilder() throws Exception {
+ SSLContextBuilder builder = new SSLContextBuilder();
+ builder.loadTrustMaterial(null, (chain, authType) -> true);
+ SSLConnectionSocketFactory sslsf = new
+ SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
+ return HttpClients.custom().setSSLSocketFactory(sslsf).build();
+ }
+
+ public static boolean checkVersionConstraint(String solrVersion, String minInclusive, String maxInclusive) {
+ String constraint = ">="+minInclusive + " & <="+maxInclusive;
+ System.out.println("Current: "+solrVersion+", constraint: "+constraint);
+ return Strings.isNullOrEmpty(constraint) || Version.valueOf(solrVersion).satisfies(constraint);
+ }
+
+ public static int compareVersions(String v1, String v2) {
+ return Version.valueOf(v1).compareTo(Version.valueOf(v2));
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java
new file mode 100644
index 0000000..3ce7336
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackage.java
@@ -0,0 +1,107 @@
+package org.apache.solr.packagemanager;
+
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Describes a package (along with all released versions) as it appears in a repository.
+ */
+public class SolrPackage implements Serializable, Comparable<SolrPackage> {
+
+ public String id;
+ public String description;
+ public List<SolrPackageRelease> versions;
+
+ private String repositoryId;
+
+ public static class SolrPackageRelease {
+
+ public String version;
+ public Date date;
+ public String url;
+
+ public String sha512sum;
+ public String sig;
+
+ public Metadata metadata;
+ @Override
+ public String toString() {
+ return "SolrPackageRelease{" +
+ "version='" + version + '\'' +
+ ", date=" + date +
+ ", url='" + url + '\'' +
+ ", sig='" + sig + '\'' +
+ ", min='" + metadata.minSolrVersion + '\'' +
+ ", max='" + metadata.maxSolrVersion + '\'' +
+ ", dependencies='" + metadata.dependencies + '\'' +
+ ", plugins='" + metadata.plugins + '\'' +
+ ", paramDefaults='" + metadata.parameterDefaults + '\'' +
+ ", sha512sum='" + sha512sum + '\'' +
+ '}';
+ }
+ }
+
+ public static class Metadata {
+ @JsonProperty("min-solr-version")
+ public String minSolrVersion;
+ @JsonProperty("max-solr-version")
+ public String maxSolrVersion;
+
+ public List<String> dependencies;
+ public List<Plugin> plugins;
+ @JsonProperty("parameter-defaults")
+ public Map<String, String> parameterDefaults;
+ }
+
+ public static class Plugin {
+ public String id;
+ @JsonProperty("setup-command")
+ public String setupCommand;
+
+ @JsonProperty("update-command")
+ public String updateCommand;
+
+ @JsonProperty("uninstall-command")
+ public String uninstallCommand;
+
+ @JsonProperty("verify-command")
+ public Command verifyCommand;
+
+ @Override
+ public String toString() {
+ return id + ": {setup: "+setupCommand+", update: "+updateCommand+", uninstall: "+uninstallCommand+", verify: "+verifyCommand+"}";
+ }
+ }
+
+ @Override
+ public int compareTo(SolrPackage o) {
+ return id.compareTo(o.id);
+ }
+
+ public String getRepositoryId() {
+ return repositoryId;
+ }
+
+ public void setRepositoryId(String repositoryId) {
+ this.repositoryId = repositoryId;
+ }
+
+ public static class Command {
+ public String path;
+ public String method;
+ public Map payload;
+ public String condition;
+ public String expected;
+
+ @Override
+ public String toString() {
+ return method + " " + path + ", Payload: "+ payload+", Condition: "+condition+", expected: "+expected;
+ }
+ }
+}
+
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java
new file mode 100644
index 0000000..9e37c4e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageInstance.java
@@ -0,0 +1,42 @@
+package org.apache.solr.packagemanager;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.packagemanager.SolrPackage.Plugin;
+
+/**
+ * Describes one instance of a package as it exists in Solr when installed.
+ */
+public class SolrPackageInstance {
+ final public String name;
+ final public String description;
+ final public String version;
+ final public List<Plugin> plugins;
+ final Map<String, String> parameterDefaults;
+
+ public SolrPackageInstance(String id, String description, String version,
+ List<Plugin> plugins, Map<String, String> params) {
+ this.name = id;
+ this.description = description;
+ this.version = version;
+ this.plugins = plugins;
+ this.parameterDefaults = params;
+ }
+
+ public String getPackageName() {
+ return name;
+ }
+
+ public String getPackageDescription() {
+ return description;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public List<Plugin> getPlugins() {
+ return plugins;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
new file mode 100644
index 0000000..12fa90f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageManager.java
@@ -0,0 +1,258 @@
+package org.apache.solr.packagemanager;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.packagemanager.SolrPackage.Command;
+import org.apache.solr.packagemanager.SolrPackage.Metadata;
+import org.apache.solr.packagemanager.SolrPackage.Plugin;
+import org.apache.solr.util.SolrCLI;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.jayway.jsonpath.JsonPath;
+
+public class SolrPackageManager implements Closeable {
+
+ final String solrBaseUrl;
+ final SolrClient solrClient;
+
+ final SolrZkClient zkClient;
+ public SolrPackageManager(String solrBaseUrl, String zkHost) {
+ this.solrBaseUrl = solrBaseUrl;
+ this.solrClient = new HttpSolrClient.Builder(solrBaseUrl).build();
+ this.zkClient = new SolrZkClient(zkHost, 30000);
+ System.out.println("Done initializing a zkClient instance...");
+ }
+
+ Map<String, List<SolrPackageInstance>> packages = null;
+
+ Metadata fetchMetadata(String manifestFilePath) throws MalformedURLException, IOException {
+ String metadataJson = PackageUtils.getStringFromStream(solrBaseUrl + "/api/node/files"+manifestFilePath);
+ System.out.println("Fetched metadata blob: "+metadataJson);
+ Metadata metadata = new ObjectMapper().readValue(metadataJson, Metadata.class);
+ System.out.println("Now metadata: "+metadata);
+ return metadata;
+ }
+
+ public List<SolrPackageInstance> getPackages() throws SolrException {
+ System.out.println("Getting packages from clusterprops...");
+ List<SolrPackageInstance> ret = new ArrayList<SolrPackageInstance>();
+ packages = new HashMap<String, List<SolrPackageInstance>>();
+ try {
+ String clusterPropsJson = null;
+ Map packagesJson = null;
+
+ if (zkClient.exists("/packages.json", true) == true) {
+ clusterPropsJson = new String(zkClient.getData("/packages.json", null, null, true), "UTF-8");
+ System.out.println("clusterprops are: "+clusterPropsJson);
+ packagesJson = (Map)new ObjectMapper().
+ configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true).readValue(clusterPropsJson, Map.class).get("packages");
+ }
+
+ if (packagesJson != null) {
+ for (Object packageName: packagesJson.keySet()) {
+ List pkg = (List)packagesJson.get(packageName);
+ for (Map pkgVersion: (List<Map>)pkg) {
+ Metadata metadata = fetchMetadata(pkgVersion.get("manifest").toString());
+ List<Plugin> solrplugins = metadata.plugins;
+ SolrPackageInstance pkgInstance = new SolrPackageInstance(packageName.toString(), null,
+ pkgVersion.get("version").toString(), solrplugins, metadata.parameterDefaults);
+ List<SolrPackageInstance> list = packages.containsKey(packageName)? packages.get(packageName): new ArrayList<SolrPackageInstance>();
+ list.add(pkgInstance);
+ packages.put(packageName.toString(), list);
+ ret.add(pkgInstance);
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ if (packages == null) packages = Collections.emptyMap(); // nocommit can't happen
+ throw new SolrException(ErrorCode.BAD_REQUEST, e);
+ }
+ System.out.println("Got packages: "+ret);
+ return ret;
+ }
+
+ Map<String, String> getPackageParams(String packageName, String collection) {
+ try {
+ return (Map<String, String>)((Map)((Map)((Map)new ObjectMapper().readValue
+ (PackageUtils.get(solrBaseUrl + "/api/collections/" + collection + "/config/params/packages"), Map.class).get("response")).get("params")).get("packages")).get(packageName);
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ public boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, List<String> collections, String overrides[]) {
+ for (String collection: collections) {
+ Map<String, String> collectionParameterOverrides = isUpdate? getPackageParams(packageInstance.name, collection): new HashMap<String,String>();
+ if (overrides != null) {
+ for (String override: overrides) {
+ collectionParameterOverrides.put(override.split("=")[0], override.split("=")[1]);
+ }
+ }
+
+ // Get package params
+ try {
+ boolean packageParamsExist = ((Map)((Map)new ObjectMapper().readValue(
+ PackageUtils.get(solrBaseUrl + "/api/collections/abc/config/params/packages"), Map.class)
+ ).get("response")).containsKey("params");
+ SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params",
+ new ObjectMapper().writeValueAsString(
+ Map.of(packageParamsExist? "update": "set",
+ Map.of("packages", Map.of(packageInstance.name, collectionParameterOverrides)))));
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+
+ String paramsJson = PackageUtils.get(solrBaseUrl+"/api/collections/"+collection+"/config/params?omitHeader=true");
+ System.out.println("Before Posting param: "+paramsJson);
+
+ // Set the package version in the collection's parameters
+ try {
+ SolrCLI.postJsonToSolr(solrClient, "/api/collections/"+collection+"/config/params", "{set:{PKG_VERSIONS:{"+packageInstance.name+" : '"+(pegToLatest? "$LATEST": packageInstance.version)+"'}}}");
+ } catch (Exception ex) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, ex);
+ }
+
+ paramsJson = PackageUtils.get(solrBaseUrl+"/api/collections/"+collection+"/config/params?omitHeader=true");
+ System.out.println("Posted param: "+paramsJson);
+
+ // If updating, refresh the package version for this to take effect
+ if (isUpdate || pegToLatest) {
+ try {
+ SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\" : \""+packageInstance.name+"\"}");
+ } catch (Exception ex) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, ex);
+ }
+ }
+
+ // Setup/update all the plugins in the package
+ for (Plugin p: packageInstance.getPlugins()) {
+ System.out.println(isUpdate? p.updateCommand: p.setupCommand);
+
+ Map<String, String> systemParams = new HashMap<String,String>();
+ systemParams.put("collection", collection);
+ systemParams.put("package-name", packageInstance.name);
+ systemParams.put("package-version", packageInstance.version);
+
+ String cmd = resolve(isUpdate? p.updateCommand: p.setupCommand, packageInstance.parameterDefaults, collectionParameterOverrides, systemParams);
+ if (cmd != null && !"".equals(cmd.trim())) {
+ try {
+ System.out.println("Executing " + cmd + " for collection:" + collection);
+ SolrCLI.postJsonToSolr(solrClient, "/solr/"+collection+"/config", cmd);
+ } catch (Exception ex) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, ex);
+ }
+ }
+ }
+
+ // Set the package version in the collection's parameters
+ try {
+ SolrCLI.postJsonToSolr(solrClient, "/api/collections/"+collection+"/config/params", "{update:{PKG_VERSIONS:{'"+packageInstance.name+"' : '"+(pegToLatest? "$LATEST": packageInstance.version)+"'}}}");
+ paramsJson = PackageUtils.get(solrBaseUrl+"/api/collections/"+collection+"/config/params?omitHeader=true");
+ System.out.println("Posted param: "+paramsJson);
+ } catch (Exception ex) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, ex);
+ }
+
+ }
+
+ // Verify that package was successfully deployed
+ boolean success = verify(packageInstance, collections);
+ if (success) {
+ System.out.println("Deployed and verified package: "+packageInstance.name+", version: "+packageInstance.version);
+ }
+ return success;
+ }
+
+ private String resolve(String str, Map<String, String> defaults, Map<String, String> overrides, Map<String, String> systemParams) {
+ if (str == null) return null;
+ for (String param: defaults.keySet()) {
+ str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param));
+ }
+ for (String param: overrides.keySet()) {
+ str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param));
+ }
+ for (String param: systemParams.keySet()) {
+ str = str.replaceAll("\\$\\{"+param+"\\}", systemParams.get(param));
+ }
+ return str;
+ }
+ //nocommit should this be private?
+ public boolean verify(SolrPackageInstance pkg, List<String> collections) {
+ // verify deployment succeeded?
+ boolean success = true;
+ for (Plugin p: pkg.getPlugins()) {
+ System.out.println(p.verifyCommand);
+ for (String collection: collections) {
+ System.out.println("Executing " + p.verifyCommand + " for collection:" + collection);
+ Map<String, String> collectionParameterOverrides = getPackageParams(pkg.name, collection);
+
+ Command cmd = p.verifyCommand;
+
+ Map<String, String> systemParams = new HashMap<String,String>();
+ systemParams.put("collection", collection);
+ systemParams.put("package-name", pkg.name);
+ systemParams.put("package-version", pkg.version);
+ String url = solrBaseUrl + resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
+
+ if ("GET".equalsIgnoreCase(cmd.method)) {
+ String response = PackageUtils.get(url);
+ System.out.println(response);
+ String actualValue = JsonPath.parse(response).read(resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams));
+ String expectedValue = resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
+ System.out.println("Actual: "+actualValue+", expected: "+expectedValue);
+ if (!expectedValue.equals(actualValue)) {
+ System.out.println("Failed to deploy plugin: "+p.id);
+ success = false;
+ }
+ } // commit POST?
+ }
+ }
+ return success;
+ }
+
+ // nocommit: javadocs should mention that version==null or "latest" will return latest version installed
+ public SolrPackageInstance getPackageInstance(String packageName, String version) {
+ getPackages();
+ List<SolrPackageInstance> versions = packages.get(packageName);
+ String latestVersion = "0.0.0";
+ SolrPackageInstance latest = null;
+ if (versions != null) {
+ for (SolrPackageInstance pkg: versions) {
+ if (pkg.version.equals(version)) {
+ return pkg;
+ }
+ if (PackageUtils.compareVersions(latestVersion, pkg.version) <= 0) {
+ latestVersion = pkg.version;
+ latest = pkg;
+ }
+ }
+ }
+ if (version == null || version.equalsIgnoreCase("latest")) {
+ return latest;
+ } else return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ if (solrClient != null) {
+ solrClient.close();
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageRepository.java
new file mode 100644
index 0000000..cf9a621
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrPackageRepository.java
@@ -0,0 +1,130 @@
+package org.apache.solr.packagemanager;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class SolrPackageRepository {
+ private static final Logger log = LoggerFactory.getLogger(SolrPackageRepository.class);
+
+ @JsonProperty("id")
+ public String id;
+ @JsonProperty("url")
+ public String url;
+
+ public SolrPackageRepository() {
+ }//nocommit wtf?
+
+ public SolrPackageRepository(String id, String url) {
+ this.id = id;
+ this.url = url;
+ }
+
+ public void refresh() {
+ packages = null;
+ }
+
+ @JsonIgnore
+ private Map<String, SolrPackage> packages;
+
+ public Map<String, SolrPackage> getPackages() {
+ if (packages == null) {
+ initPackages();
+ }
+
+ return packages;
+ }
+
+ public SolrPackage getPackage(String id) {
+ return getPackages().get(id);
+ }
+
+ public boolean hasPackage(String packageName) {
+ return getPackages().containsKey(packageName);
+ }
+
+ public Path download(URL url) throws SolrException, IOException {
+ Path tmpDirectory = Files.createTempDirectory("solr-packages");
+ tmpDirectory.toFile().deleteOnExit();
+ String fileName = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
+ Path destination = tmpDirectory.resolve(fileName);
+
+ switch (url.getProtocol()) {
+ case "http":
+ case "https":
+ case "ftp":
+ FileUtils.copyURLToFile(url, destination.toFile());
+ break;
+ case "file":
+ try {
+ FileUtils.copyFile(new File(url.toURI()), destination.toFile());
+ } catch (URISyntaxException e) {
+ throw new SolrException(ErrorCode.INVALID_STATE, e);
+ }
+ break;
+ default:
+ throw new SolrException(ErrorCode.BAD_REQUEST, "URL protocol " + url.getProtocol() + " not supported");
+ }
+
+ return destination;
+ }
+
+ private void initPackages() {
+ Reader pluginsJsonReader;
+ try {
+ URL pluginsUrl = new URL(new URL(url), "manifest.json"); //nocommit hardcoded
+ log.debug("Read plugins of '{}' repository from '{}'", id, pluginsUrl);
+ pluginsJsonReader = new InputStreamReader(pluginsUrl.openStream());
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ packages = Collections.emptyMap();
+ return;
+ }
+
+ ObjectMapper mapper = new ObjectMapper();
+ SolrPackage items[];
+ try {
+ items = mapper.readValue(pluginsJsonReader, SolrPackage[].class);
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
+ packages = new HashMap<>(items.length);
+ for (SolrPackage p : items) {
+ for (SolrPackageRelease r : p.versions) {
+ try {
+ r.url = new URL(new URL(url), r.url).toString();
+ if (r.date.getTime() == 0) {
+ log.warn("Illegal release date when parsing {}@{}, setting to epoch", p.id, r.version);
+ }
+ } catch (MalformedURLException e) {
+ log.warn("Skipping release {} of plugin {} due to failure to build valid absolute URL. Url was {}{}", r.version, p.id, url, r.url);
+ }
+ }
+ p.setRepositoryId(id);
+ packages.put(p.id, p);
+
+ System.out.println("****\n"+p+"\n*******");
+ }
+ log.debug("Found {} plugins in repository '{}'", packages.size(), id);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java b/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
new file mode 100644
index 0000000..0ab355d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/packagemanager/SolrUpdateManager.java
@@ -0,0 +1,218 @@
+package org.apache.solr.packagemanager;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.util.Version;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.request.beans.Package;
+import org.apache.solr.client.solrj.response.V2Response;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class SolrUpdateManager {
+
+ final private SolrPackageManager packageManager;
+ final private String repositoriesJsonStr;
+ protected List<SolrPackageRepository> repositories;
+
+ public static final String systemVersion = Version.LATEST.toString();
+
+ final String solrBaseUrl;
+
+ private static final Logger log = LoggerFactory.getLogger(SolrUpdateManager.class);
+
+ public SolrUpdateManager(SolrPackageManager packageManager, String repositoriesJsonStr, String solrBaseUrl) {
+ this.packageManager = packageManager;
+ this.repositoriesJsonStr = repositoriesJsonStr;
+ this.solrBaseUrl = solrBaseUrl;
+ }
+
+ protected synchronized void initRepositoriesFromJson() {
+ SolrPackageRepository items[];
+ try {
+ items = new ObjectMapper().readValue(this.repositoriesJsonStr, SolrPackageRepository[].class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ this.repositories = Arrays.asList(items);
+ }
+
+ public synchronized void refresh() {
+ initRepositoriesFromJson();
+ for (SolrPackageRepository updateRepository : repositories) {
+ updateRepository.refresh();
+ }
+ }
+
+ // nocommit do we need this, when we have a map version of this?
+ public List<SolrPackage> getPackages() {
+ List<SolrPackage> list = new ArrayList<>(getPackagesMap().values());
+ Collections.sort(list);
+
+ return list;
+ }
+
+ public Map<String, SolrPackage> getPackagesMap() {
+ Map<String, SolrPackage> packagesMap = new HashMap<>();
+ for (SolrPackageRepository repository : getRepositories()) {
+ packagesMap.putAll(repository.getPackages());
+ }
+
+ return packagesMap;
+ }
+
+ public List<SolrPackageRepository> getRepositories() {
+ refresh();
+ return repositories;
+ }
+
+ public boolean updateOrInstallPackage(String packageName, String version) throws SolrException {
+ // nocommit: handle version being null
+ SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version);
+ if (existingPlugin != null && existingPlugin.getVersion().equals(version)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Plugin already installed.");
+ }
+
+ SolrPackage pkg = getPackagesMap().get(packageName);
+ SolrPackageRelease release = findReleaseForPackage(packageName, version);
+ Path downloaded = downloadPackage(packageName, version);
+
+ try (HttpSolrClient solrClient = new HttpSolrClient.Builder(solrBaseUrl).build()) {
+ // post the metadata
+ System.out.println("Posting metadata");
+ PackageUtils.postFile(solrClient, ByteBuffer.wrap(new ObjectMapper().writeValueAsString(release.metadata).getBytes()),
+ "/package/"+packageName+"/"+version+"/solr-manifest.json",
+ null);
+
+ // post the artifacts
+ System.out.println("Posting artifacts");
+ PackageUtils.postFile(solrClient, PackageUtils.getFileContent(downloaded.toFile()),
+ "/package/"+packageName+"/"+version+"/"+downloaded.getFileName().toString(),
+ release.sig
+ );
+
+ // Call Package Manager API to add this version of the package
+ Package.AddVersion add = new Package.AddVersion();
+ add.version = version;
+ add.pkg = packageName;
+ add.files = Arrays.asList(new String[] {"/package/"+packageName+"/"+version+"/"+downloaded.getFileName().toString()});
+ add.manifest = "/package/" + packageName + "/" + version + "/solr-manifest.json";
+ add.manifestSHA512 = "MY_MANIFEST_SHA512"; // nocommit: hardcoded sha512 of manifest must go away
+
+ V2Request req = new V2Request.Builder("/api/cluster/package")
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
+
+ try {
+ V2Response resp = req.process(solrClient);
+ System.out.println("Response: "+resp.jsonStr());
+ } catch (SolrServerException | IOException e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, e);
+ }
+
+ } catch (SolrServerException | IOException e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, e);
+ }
+ return false;
+ }
+
+ protected Path downloadPackage(String id, String version) throws SolrException {
+ try {
+ SolrPackageRelease release = findReleaseForPackage(id, version);
+
+ for (SolrPackageRepository repo: repositories) {
+ if (repo.hasPackage(id)) {
+ return repo.download(new URL(release.url));
+ }
+ }
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Error during download of plugin " + id, e);
+ }
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Package not found in any repository.");
+ }
+
+ public SolrPackageRelease findReleaseForPackage(String id, String version) throws SolrException {
+ SolrPackage pkg = getPackagesMap().get(id);
+ if (pkg == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Package "+id+" not found in any repository");
+ }
+
+ if (version == null || "latest".equals(version)) {
+ return getLastPackageRelease(pkg);
+ }
+
+ for (SolrPackageRelease release : pkg.versions) {
+ if (PackageUtils.compareVersions(version, release.version) == 0 && release.url != null) {
+ return release;
+ }
+ }
+
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Package "+id+" with version @"+version+" does not exist in the repository");
+ }
+
+ public SolrPackageRelease getLastPackageRelease(String id) {
+ SolrPackage pkg = getPackagesMap().get(id);
+ if (pkg == null) {
+ return null;
+ }
+ return getLastPackageRelease(pkg);
+ }
+
+ public SolrPackageRelease getLastPackageRelease(SolrPackage pkg) {
+ SolrPackageRelease latest = null;
+ for (SolrPackageRelease release: pkg.versions) {
+ if (latest == null) {
+ latest = release;
+ } else {
+ if (PackageUtils.compareVersions(latest.version, release.version) < 0) {
+ latest = release;
+ }
+ }
+ }
+ return latest;
+ }
+
+ public boolean hasPackageUpdate(String id) {
+ SolrPackage pkg = getPackagesMap().get(id);
+ if (pkg == null) {
+ return false;
+ }
+ String installedVersion = packageManager.getPackageInstance(id, null).getVersion();
+ SolrPackageRelease last = getLastPackageRelease(id);
+ return last != null && PackageUtils.compareVersions(last.version, installedVersion) > 0;
+ }
+
+ public List<SolrPackage> getUpdates() {
+ List<SolrPackage> updates = new ArrayList<>();
+ for (SolrPackageInstance installed : packageManager.getPackages()) {
+ String packageName = installed.getPackageName();
+ if (hasPackageUpdate(packageName)) {
+ updates.add(getPackagesMap().get(packageName));
+ }
+ }
+ return updates;
+ }
+
+ public boolean hasUpdates() {
+ return getUpdates().size() > 0;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index bf29b64..bf48d34 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -175,12 +175,20 @@ public class PackageAPI {
@JsonProperty
public List<String> files;
+ @JsonProperty
+ public String manifest;
+
+ @JsonProperty
+ public String manifestSHA512;
+
public PkgVersion() {
}
public PkgVersion(Package.AddVersion addVersion) {
this.version = addVersion.version;
this.files = addVersion.files;
+ this.manifest = addVersion.manifest;
+ this.manifestSHA512 = addVersion.manifestSHA512;
}
diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java
new file mode 100644
index 0000000..aa3f4a4
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.packagemanager.PackageUtils;
+import org.apache.solr.packagemanager.SolrPackage;
+import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
+import org.apache.solr.packagemanager.SolrPackageInstance;
+import org.apache.solr.packagemanager.SolrPackageManager;
+import org.apache.solr.packagemanager.SolrPackageRepository;
+import org.apache.solr.packagemanager.SolrUpdateManager;
+import org.apache.solr.util.SolrCLI.StatusTool;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
+
+
+public class PackageTool extends SolrCLI.ToolBase {
+ @Override
+ public String getName() {
+ return "package";
+ }
+
+ public static String solrUrl = null;
+ public static String solrBaseUrl = null;
+
+ public SolrPackageManager packageManager;
+ public SolrUpdateManager updateManager;
+
+ @Override
+ protected void runImpl(CommandLine cli) throws Exception {
+ // Need a logging free, clean output going through to the user.
+ Configurator.setRootLevel(Level.INFO);
+
+ solrUrl = cli.getOptionValues("solrUrl")[cli.getOptionValues("solrUrl").length-1];
+ solrBaseUrl = solrUrl.replaceAll("\\/solr$", ""); // strip out ending "/solr"
+ System.out.println("solr url: "+solrUrl+", solr base url: "+solrBaseUrl);
+
+ String zkHost = getZkHost(cli);
+
+ System.out.println("ZK: "+zkHost);
+ String cmd = cli.getArgs()[0];
+
+ try (SolrZkClient zkclient = new SolrZkClient(zkHost, 30000)) {
+ if (cmd != null) {
+ packageManager = new SolrPackageManager(solrBaseUrl, zkHost);
+ try {
+ updateManager = new SolrUpdateManager(packageManager,
+ getRepositoriesJson(zkclient), solrBaseUrl);
+
+ switch (cmd) {
+ case "add-repo":
+ addRepo(zkHost, cli.getArgs()[1], cli.getArgs()[2]);
+ break;
+ case "list-installed":
+ list(cli.getArgList().subList(1, cli.getArgList().size()));
+ break;
+ case "list-available":
+ try {
+ available(cli.getArgList().subList(1, cli.getArgList().size()));
+ } catch (SolrException ex) {
+ ex.printStackTrace();
+ }
+ break;
+ case "install":
+ install(cli.getArgList().subList(1, cli.getArgList().size()));
+ break;
+ case "deploy":
+ String colls[] = cli.getOptionValues("collections");
+ String params[] = cli.getOptionValues("param");
+ System.out.println("coll: "+Arrays.toString(colls)+", params: "+Arrays.toString(params));
+ String packageName = cli.getArgList().get(1).toString().split(":")[0];
+ String version = cli.getArgList().get(1).toString().contains(":")?
+ cli.getArgList().get(1).toString().split(":")[1]: null;
+ deploy(packageName, version, cli.hasOption("update"), colls, params);
+ break;
+ case "update":
+ if (cli.getArgList().size() == 1) {
+ update();
+ } else {
+ updatePackage(zkHost, cli.getArgs()[1], cli.getArgList().subList(2, cli.getArgList().size()));
+ }
+ break;
+ default:
+ throw new RuntimeException("Unrecognized command: "+cmd);
+ };
+ } finally {
+ packageManager.close();
+ }
+ }
+ }
+ System.out.println("Finished: "+cmd); // nocommit
+ }
+
+ protected void addRepo(String zkHost, String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException {
+ try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
+ String existingRepositoriesJson = getRepositoriesJson(zkClient);
+ System.out.println(existingRepositoriesJson);
+
+ List repos = new ObjectMapper().readValue(existingRepositoriesJson, List.class);
+ repos.add(new SolrPackageRepository(name, uri));
+ if (zkClient.exists("/repositories.json", true) == false) {
+ zkClient.create("/repositories.json", new ObjectMapper().writeValueAsString(repos).getBytes(), CreateMode.PERSISTENT, true);
+ } else {
+ zkClient.setData("/repositories.json", new ObjectMapper().writeValueAsString(repos).getBytes(), true);
+ }
+
+ if (zkClient.exists("/keys", true)==false) zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true);
+ if (zkClient.exists("/keys/exe", true)==false) zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true);
+ if (zkClient.exists("/keys/exe/"+"pub_key.der", true)==false) zkClient.create("/keys/exe/"+"pub_key.der", new byte[0], CreateMode.PERSISTENT, true);
+ zkClient.setData("/keys/exe/"+"pub_key.der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true);
+
+ System.out.println("Added repository: "+name);
+ System.out.println(getRepositoriesJson(zkClient));
+ }
+ }
+
+ protected String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException {
+ if (zkClient.exists("/repositories.json", true)) {
+ return new String(zkClient.getData("/repositories.json", null, null, true), "UTF-8");
+ }
+ return "[]";
+ }
+
+ protected void list(List args) {
+ for (SolrPackageInstance pkg: packageManager.getPackages()) {
+ System.out.println(pkg.getPackageName()+" ("+pkg.getVersion()+")");
+ }
+ }
+ protected void available(List args) throws SolrException {
+ System.out.println("Available packages:\n-----");
+ for (SolrPackage i: updateManager.getPackages()) {
+ SolrPackage plugin = (SolrPackage)i;
+ System.out.println(plugin.id + " \t\t"+plugin.description);
+ for (SolrPackageRelease version: plugin.versions) {
+ System.out.println("\tVersion: "+version.version);
+ }
+ }
+ }
+ protected void install(List args) throws SolrException {
+ String pkg = args.get(0).toString().split(":")[0];
+ String version = args.get(0).toString().contains(":")? args.get(0).toString().split(":")[1]: null;
+ updateManager.updateOrInstallPackage(pkg, version);
+ System.out.println(args.get(0).toString() + " installed.");
+ }
+ protected void deploy(String packageName, String version, boolean isUpdate,
+ String collections[], String parameters[]) throws SolrException {
+
+ boolean pegToLatest = "latest".equals(version); // User wants to peg this package's version to the latest installed (for auto-update, i.e. no explicit deploy step)
+ SolrPackageInstance packageInstance = packageManager.getPackageInstance(packageName, version);
+ // nocommit if not found, exception here
+ if (version == null) version = packageInstance.getVersion();
+
+ SolrPackageRelease release = updateManager.findReleaseForPackage(packageName, version); // TODO: If we could store the min and max in ZK itself, we wouldn't have had to look up the repository for this info.
+ if (release == null) {
+ System.out.println("Release not found for "+packageName+", version "+version);
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Release not found for "+packageName+", version "+version);
+ }
+ if (PackageUtils.checkVersionConstraint(SolrUpdateManager.systemVersion, release.metadata.minSolrVersion, release.metadata.maxSolrVersion) == false) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Version incompatible! Solr version: "
+ + SolrUpdateManager.systemVersion+", package minSolrVersion: " + release.metadata.minSolrVersion
+ + ", maxSolrVersion: "+release.metadata.maxSolrVersion);
+ }
+
+
+ System.out.println(packageManager.deployPackage(packageInstance, pegToLatest, isUpdate,
+ Arrays.asList(collections), parameters));
+ }
+
+ protected void update() throws SolrException {
+ if (updateManager.hasUpdates()) {
+ System.out.println("Available updates:\n-----");
+
+ for (SolrPackage i: updateManager.getUpdates()) {
+ SolrPackage plugin = (SolrPackage)i;
+ System.out.println(plugin.id + " \t\t"+plugin.description);
+ for (SolrPackageRelease version: plugin.versions) {
+ System.out.println("\tVersion: "+version.version);
+ }
+ }
+ } else {
+ System.out.println("No updates found. System is up to date.");
+ }
+ }
+
+ protected void updatePackage(String zkHost, String packageName, List args) throws SolrException {
+ if (updateManager.hasUpdates()) {
+ String latestVersion = updateManager.getLastPackageRelease(packageName).version;
+ SolrPackageInstance installedPackage = packageManager.getPackageInstance(packageName, "latest");
+ System.out.println("Updating ["+packageName+"] from " + installedPackage.getVersion() + " to version "+latestVersion);
+
+ List<String> collectionsDeployedIn = getDeployedCollections(zkHost, packageManager, installedPackage, true);
+ System.out.println("Already deployed on collections: "+collectionsDeployedIn);
+ updateManager.updateOrInstallPackage(packageName, latestVersion);
+
+ SolrPackageInstance updatedPackage = packageManager.getPackageInstance(packageName, "latest");
+ boolean res = packageManager.verify(updatedPackage, collectionsDeployedIn);
+ System.out.println("Verifying version "+updatedPackage.getVersion()+" on "+collectionsDeployedIn
+ +", result: "+res);
+ if (!res) throw new SolrException(ErrorCode.BAD_REQUEST, "Failed verification after deployment");
+ } else {
+ System.out.println("Package "+packageName+" is already up to date.");
+ }
+ }
+
+ private List<String> getDeployedCollections(String zkHost, SolrPackageManager packageManager, SolrPackageInstance pkg, boolean onlyLatest) {
+
+ List<String> allCollections;
+ try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
+ allCollections = zkClient.getChildren("/collections", null, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ System.out.println("Need to verify if these collections have the plugin installed? "+ allCollections);
+ List<String> deployed = new ArrayList<String>();
+ for (String collection: allCollections) {
+ // Check package version installed
+ // http://localhost:8983/api/collections/abc/config/params/PKG_VERSIONS?omitHeader=true
+ String paramsJson = PackageUtils.get(solrBaseUrl+"/api/collections/"+collection+"/config/params/PKG_VERSIONS?omitHeader=true");
+ String version = null;
+ try {
+ version = JsonPath.parse(paramsJson).read("$['response'].['params'].['PKG_VERSIONS'].['"+pkg.name+"'])");
+ } catch (PathNotFoundException ex) {
+ // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager.
+ }
+ if (onlyLatest) {
+ if ("$LATEST".equals(version) && packageManager.verify(pkg, Collections.singletonList(collection))) {
+ deployed.add(collection);
+ } else {
+ System.out.println("Skipping collection: "+collection+", version: "+version);
+ }
+ } else {
+ if (packageManager.verify(pkg, Collections.singletonList(collection))) {
+ deployed.add(collection);
+ }
+ }
+ }
+ return deployed;
+ }
+
+ // nocommit fix the descriptions of all the options
+ @SuppressWarnings("static-access")
+ public Option[] getOptions() {
+ return new Option[] {
+ OptionBuilder
+ .withArgName("URL")
+ .hasArg()
+ .isRequired(true)
+ .withDescription("Address of the Solr Web application, defaults to: "+SolrCLI.DEFAULT_SOLR_URL)
+ .create("solrUrl"),
+
+ OptionBuilder
+ .withArgName("COLLECTIONS")
+ .hasArgs()
+ .isRequired(false)
+ .withDescription("Solr URL scheme: http or https, defaults to http if not specified")
+ .create("collections"),
+
+ OptionBuilder
+ .withArgName("PARAMS")
+ .hasArgs()
+ .isRequired(false)
+ .withDescription("Solr URL scheme: http or https, defaults to http if not specified")
+ .withLongOpt("param")
+ .create("p"),
+
+ OptionBuilder
+ .isRequired(false)
+ .withDescription("Solr URL scheme: http or https, defaults to http if not specified")
+ .withLongOpt("update")
+ .create("u"),
+
+ OptionBuilder
+ .isRequired(false)
+ .withDescription("Solr URL scheme: http or https, defaults to http if not specified")
+ .withLongOpt("auto-update")
+ .create(),
+
+ };
+ }
+
+ private String getZkHost(CommandLine cli) throws Exception {
+ String zkHost = cli.getOptionValue("zkHost");
+ if (zkHost != null)
+ return zkHost;
+
+ // find it using the localPort
+
+ String systemInfoUrl = solrUrl+"/admin/info/system";
+ CloseableHttpClient httpClient = SolrCLI.getHttpClient();
+ try {
+ // hit Solr to get system info
+ Map<String,Object> systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true);
+
+ // convert raw JSON into user-friendly output
+ StatusTool statusTool = new StatusTool();
+ Map<String,Object> status = statusTool.reportStatus(solrUrl+"/", systemInfo, httpClient);
+ Map<String,Object> cloud = (Map<String, Object>)status.get("cloud");
+ if (cloud != null) {
+ String zookeeper = (String) cloud.get("ZooKeeper");
+ if (zookeeper.endsWith("(embedded)")) {
+ zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length());
+ }
+ zkHost = zookeeper;
+ }
+ } finally {
+ HttpClientUtil.close(httpClient);
+ }
+
+ return zkHost;
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
index d077d7d..3feb0da 100755
--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
@@ -419,6 +419,8 @@ public class SolrCLI implements CLIO {
return new AutoscalingTool();
else if ("export".equals(toolType))
return new ExportTool();
+ else if ("package".equals(toolType))
+ return new PackageTool();
// If you add a built-in tool to this class, add it here to avoid
// classpath scanning
diff --git a/solr/core/src/test-files/solr/question-answer-repository/manifest.json b/solr/core/src/test-files/solr/question-answer-repository/manifest.json
new file mode 100644
index 0000000..31998d8
--- /dev/null
+++ b/solr/core/src/test-files/solr/question-answer-repository/manifest.json
@@ -0,0 +1,65 @@
+[
+ {
+ "id": "question-answer",
+ "description": "A natural language question answering plugin",
+ "versions": [
+ {
+ "version": "1.0.0",
+ "date": "2019-01-01",
+ "url": "question-answer-request-handler-1.0.jar",
+ "sig": "C9UWKkucmY3UNzqn0VLneVMe9kCbJjw7Urc76vGenoRwp32xvNn5ZIGZ7G34xZP7cVjqn/ltDlLWBZ/C3eAtuw==",
+ "metadata": {
+ "min-solr-version": "8.0",
+ "max-solr-version": "9.99",
+ "dependencies": [],
+ "plugins": [
+ {
+ "id": "request-handler",
+ "setup-command": "{\"add-requesthandler\": {\"name\": \"${RH-HANDLER-PATH}\", \"class\": \"question-answer:fullstory.QARequestHandler\", \"defaults\":{ \"x\": \"y\" ,\"a\": \"b\", \"rows\":10 }, \"useParams\": \"x\"}}",
+ "update-command": "{\"update-requesthandler\": {\"name\": \"${RH-HANDLER-PATH}\", \"class\": \"question-answer:fullstory.QARequestHandler\", \"defaults\":{ \"x\": \"y\" ,\"a\": \"b\", \"rows\":10 }, \"useParams\": \"x\"}}",
+ "uninstall-command": null,
+ "verify-command": {
+ "path": "/api/collections/${collection}/config/requestHandler?componentName=${RH-HANDLER-PATH}&meta=true",
+ "method": "GET",
+ "condition": "$['config'].['requestHandler'].['${RH-HANDLER-PATH}'].['_packageinfo_'].['version']",
+ "expected": "${package-version}"
+ }
+ }
+ ],
+ "parameter-defaults": {
+ "RH-HANDLER-PATH": "/mypath"
+ }
+ }
+ },
+ {
+ "version": "1.1.0",
+ "date": "2019-01-01",
+ "url": "question-answer-request-handler-1.1.jar",
+ "sig": "VKfK8aAgH8Ys8vZ0Pym3vnWalhC8WpfcwADw3pvUHG9Kui0UMJ0L6hTNuH41tuo5FrfG7cu2roPGsYXg2WzV5g==",
+ "metadata": {
+ "min-solr-version": "8.0",
+ "max-solr-version": "9.99",
+ "dependencies": [],
+ "plugins": [
+ {
+ "id": "request-handler",
+ "setup-command": "{\"add-requesthandler\": {\"name\": \"${RH-HANDLER-PATH}\", \"class\": \"question-answer:fullstory.QARequestHandler\", \"defaults\":{ \"x\": \"y\" ,\"a\": \"b\", \"rows\":10 }, \"useParams\": \"x\"}}",
+ "update-command": "{\"update-requesthandler\": {\"name\": \"${RH-HANDLER-PATH}\", \"class\": \"question-answer:fullstory.QARequestHandler\", \"defaults\":{ \"x\": \"y\" ,\"a\": \"b\", \"rows\":10 }, \"useParams\": \"x\"}}",
+ "uninstall-command": null,
+ "verify-command": {
+ "path": "/api/collections/${collection}/config/requestHandler?componentName=${RH-HANDLER-PATH}&meta=true",
+ "method": "GET",
+ "condition": "$['config'].['requestHandler'].['${RH-HANDLER-PATH}'].['_packageinfo_'].['version']",
+ "expected": "${package-version}"
+ }
+ }
+ ],
+ "parameter-defaults": {
+ "RH-HANDLER-PATH": "/mypath"
+ }
+ }
+ }
+ ]
+ }
+]
+
diff --git a/solr/core/src/test-files/solr/question-answer-repository/publickey.der b/solr/core/src/test-files/solr/question-answer-repository/publickey.der
new file mode 100644
index 0000000..8ca82e8
Binary files /dev/null and b/solr/core/src/test-files/solr/question-answer-repository/publickey.der differ
diff --git a/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar
new file mode 100644
index 0000000..46394e4
Binary files /dev/null and b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.0.jar differ
diff --git a/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar
new file mode 100644
index 0000000..1eaa2b3
Binary files /dev/null and b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.1.jar differ
diff --git a/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.2.jar b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.2.jar
new file mode 100644
index 0000000..88354b6
Binary files /dev/null and b/solr/core/src/test-files/solr/question-answer-repository/question-answer-request-handler-1.2.jar differ
diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
new file mode 100644
index 0000000..679187a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.core.TestSolrConfigHandler;
+import org.apache.solr.util.PackageTool;
+import org.apache.solr.util.SolrCLI;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.DefaultHandler;
+import org.eclipse.jetty.server.handler.HandlerList;
+import org.eclipse.jetty.server.handler.ResourceHandler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackageManagerCLITest extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static LocalWebServer repositoryServer;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ System.setProperty("enable.packages", "true");
+
+ configureCluster(1)
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
+
+ repositoryServer = new LocalWebServer(TEST_PATH().resolve("question-answer-repository").toString());
+ repositoryServer.start();
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception {
+ repositoryServer.stop();
+ }
+
+ @Test
+ public void testPackageManager() throws Exception {
+ PackageTool tool = new PackageTool();
+ String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"});
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "add-repo", "fullstory", "http://localhost:" + repositoryServer.getPort()});
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "list-available"});
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "install", "question-answer:1.0.0"}); // no-commit (change to pkg:ver syntax)
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"});
+
+ CollectionAdminRequest.createCollection("abc", "conf1", 1, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("def", "conf1", 1, 1).process(cluster.getSolrClient());
+
+ String rhPath = "/mypath2";
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath});
+ assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0");
+
+ // Should we test the "auto-update to latest" functionality or the default explicit deploy functionality
+ boolean autoUpdateToLatest = random().nextBoolean();
+
+ if (autoUpdateToLatest) {
+ log.info("Testing auto-update to latest installed");
+
+ // This command pegs the version to the latest available
+ run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "question-answer:latest", "-collections", "abc"});
+ assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.0.0");
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "update", "question-answer"});
+ assertPackageVersion("abc", "question-answer", "$LATEST", rhPath, "1.1.0");
+ } else {
+ log.info("Testing explicit deployment to a different/newer version");
+
+ run(tool, new String[] {"-solrUrl", solrUrl, "update", "question-answer"});
+ assertPackageVersion("abc", "question-answer", "1.0.0", rhPath, "1.0.0");
+
+
+ if (random().nextBoolean()) { // even if parameters are not passed in, they should be picked up from previous deployment
+ run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "question-answer", "-collections", "abc", "-p", "RH-HANDLER-PATH=" + rhPath});
+ } else {
+ run(tool, new String[] {"-solrUrl", solrUrl, "deploy", "--update", "question-answer", "-collections", "abc"});
+ }
+ assertPackageVersion("abc", "question-answer", "1.1.0", rhPath, "1.1.0");
+ }
+ }
+
+ void assertPackageVersion(String collection, String pkg, String version, String component, String componentVersion) throws Exception {
+ TestSolrConfigHandler.testForResponseElement(
+ null,
+ cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection,
+ "/config/params?meta=true",
+ cluster.getSolrClient(),
+ Arrays.asList("response", "params", "PKG_VERSIONS", pkg),
+ version,
+ 1);
+
+ TestSolrConfigHandler.testForResponseElement(
+ null,
+ cluster.getJettySolrRunner(0).getBaseUrl().toString() + "/" + collection,
+ "/config/requestHandler?componentName=" + component + "&meta=true",
+ cluster.getSolrClient(),
+ Arrays.asList("config", "requestHandler", component, "_packageinfo_", "version"),
+ componentVersion,
+ 1);
+ }
+
+ private void run(PackageTool tool, String[] args) throws Exception {
+ int res = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args));
+ assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res);
+ }
+
+ static class LocalWebServer {
+ private int port = 0;
+ final private String resourceBase;
+ Server server;
+ ServerConnector connector;
+
+ public LocalWebServer(String resourceDir) {
+ this.resourceBase = resourceDir;
+ }
+
+ public int getPort() {
+ return connector != null? connector.getLocalPort(): port;
+ }
+
+ public LocalWebServer setPort(int port) {
+ this.port = port;
+ return this;
+ }
+
+ public String getResourceBase() {
+ return resourceBase;
+ }
+
+ public void start() throws Exception {
+ server = new Server();
+
+ connector = new ServerConnector(server);
+ connector.setPort(port);
+ server.addConnector(connector);
+ server.setStopAtShutdown(true);
+
+ ResourceHandler resourceHandler = new ResourceHandler();
+ resourceHandler.setResourceBase(resourceBase);
+ resourceHandler.setDirectoriesListed(true);
+
+ HandlerList handlers = new HandlerList();
+ handlers.setHandlers(new Handler[] { resourceHandler, new DefaultHandler() });
+ server.setHandler(handlers);
+
+ server.start();
+ }
+
+ public void stop() throws Exception {
+ server.stop();
+ }
+ }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java
index bda1078..d3fe647 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java
@@ -33,6 +33,10 @@ public class Package {
public String version;
@JsonProperty(required = true)
public List<String> files;
+ @JsonProperty
+ public String manifest;
+ @JsonProperty
+ public String manifestSHA512;
}