You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/07/31 08:42:48 UTC
[lucene-solr] branch branch_8x updated: SOLR-14681: Introduce
ability to delete .jar stored in the Package Store
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 5001568 SOLR-14681: Introduce ability to delete .jar stored in the Package Store
5001568 is described below
commit 50015687bfee4486a94368bef81b8456c2bf1d79
Author: Marcus <ma...@gmail.com>
AuthorDate: Fri Jul 31 01:23:18 2020 -0700
SOLR-14681: Introduce ability to delete .jar stored in the Package Store
---
solr/CHANGES.txt | 2 +
.../apache/solr/filestore/DistribPackageStore.java | 67 +++++++--
.../org/apache/solr/filestore/PackageStore.java | 7 +
.../org/apache/solr/filestore/PackageStoreAPI.java | 47 +++++++
.../src/java/org/apache/solr/pkg/PackageAPI.java | 19 ++-
.../solr/filestore/TestDistribPackageStore.java | 63 +++++----
.../src/test/org/apache/solr/pkg/TestPackages.java | 155 +++++++++++----------
.../java/org/apache/solr/common/util/Utils.java | 68 ++++-----
8 files changed, 275 insertions(+), 153 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0dae8de..488db18 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -13,6 +13,8 @@ New Features
* SOLR-14151 Make schema components load from packages (noble)
+* SOLR-14681: Introduce ability to delete .jar stored in the Package Store. (MarcusSorealheis , Mike Drob)
+
Improvements
---------------------
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index f6f793b..e6024b9 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -39,8 +39,11 @@ import java.util.function.Consumer;
import java.util.function.Predicate;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpDelete;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Utils;
@@ -90,7 +93,7 @@ public class DistribPackageStore implements PackageStore {
path = File.separator + path;
}
return new File(solrHome +
- File.separator + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath();
+ File.separator + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath();
}
class FileInfo {
@@ -182,8 +185,8 @@ public class DistribPackageStore implements PackageStore {
Map m = null;
try {
metadata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
- baseUrl + "/node/files" + getMetaPath(),
- Utils.newBytesConsumer((int) MAX_PKG_SIZE));
+ baseUrl + "/node/files" + getMetaPath(),
+ Utils.newBytesConsumer((int) MAX_PKG_SIZE));
m = (Map) Utils.fromJSON(metadata.array(), metadata.arrayOffset(), metadata.limit());
} catch (SolrException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching metadata", e);
@@ -191,8 +194,8 @@ public class DistribPackageStore implements PackageStore {
try {
ByteBuffer filedata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
- baseUrl + "/node/files" + path,
- Utils.newBytesConsumer((int) MAX_PKG_SIZE));
+ baseUrl + "/node/files" + path,
+ Utils.newBytesConsumer((int) MAX_PKG_SIZE));
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(filedata));
String expected = (String) m.get("sha512");
if (!sha512.equals(expected)) {
@@ -216,7 +219,7 @@ public class DistribPackageStore implements PackageStore {
String baseurl = stateReader.getBaseUrlForNodeName(liveNode);
String url = baseurl.replace("/solr", "/api");
String reqUrl = url + "/node/files" + path +
- "?meta=true&wt=javabin&omitHeader=true";
+ "?meta=true&wt=javabin&omitHeader=true";
boolean nodeHasBlob = false;
Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER);
if (Utils.getObjectByPath(nl, false, Arrays.asList("files", path)) != null) {
@@ -331,7 +334,7 @@ public class DistribPackageStore implements PackageStore {
String dirName = info.path.substring(0, info.path.lastIndexOf('/'));
coreContainer.getZkController().getZkClient().makePath(ZK_PACKAGESTORE + dirName, false, true);
coreContainer.getZkController().getZkClient().create(ZK_PACKAGESTORE + info.path, info.getDetails().getMetaData().sha512.getBytes(UTF_8),
- CreateMode.PERSISTENT, true);
+ CreateMode.PERSISTENT, true);
} catch (Exception e) {
throw new SolrException(SERVER_ERROR, "Unable to create an entry in ZK", e);
}
@@ -369,7 +372,7 @@ public class DistribPackageStore implements PackageStore {
//fire and forget
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), url, null);
} catch (Exception e) {
- log.info("Node: {} failed to respond for file fetch notification", node, e);
+ log.info("Node: {} failed to respond for file fetch notification", node, e);
//ignore the exception
// some nodes may be down or not responding
}
@@ -471,12 +474,46 @@ public class DistribPackageStore implements PackageStore {
}
@Override
+ public void delete(String path) {
+ deleteLocal(path);
+ List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
+ HttpClient client = coreContainer.getUpdateShardHandler().getDefaultHttpClient();
+ for (String node : nodes) {
+ String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node);
+ String url = baseUrl.replace("/solr", "/api") + "/node/files" + path;
+ HttpDelete del = new HttpDelete(url);
+ coreContainer.runAsync(() -> Utils.executeHttpMethod(client, url, null, del));//invoke delete command on all nodes asynchronously
+ }
+ }
+
+ private void checkInZk(String path) {
+ try {
+ //fail if file exists
+ if (coreContainer.getZkController().getZkClient().exists(ZK_PACKAGESTORE + path, true)) {
+ throw new SolrException(BAD_REQUEST, "The path exist ZK, delete and retry");
+ }
+
+ } catch (SolrException se) {
+ throw se;
+ } catch (Exception e) {
+ log.error("Could not connect to ZK", e);
+ }
+ }
+
+ @Override
+ public void deleteLocal(String path) {
+ checkInZk(path);
+ FileInfo f = new FileInfo(path);
+ f.deleteFile();
+ }
+
+ @Override
public void refresh(String path) {
try {
@SuppressWarnings({"rawtypes"})
List l = null;
try {
- l = coreContainer.getZkController().getZkClient().getChildren(ZK_PACKAGESTORE+ path, null, true);
+ l = coreContainer.getZkController().getZkClient().getChildren(ZK_PACKAGESTORE + path, null, true);
} catch (KeeperException.NoNodeException e) {
// does not matter
}
@@ -485,7 +522,7 @@ public class DistribPackageStore implements PackageStore {
List myFiles = list(path, s -> true);
for (Object f : l) {
if (!myFiles.contains(f)) {
- log.info("{} does not exist locally, downloading.. ",f);
+ log.info("{} does not exist locally, downloading.. ", f);
fetch(path + "/" + f.toString(), "*");
}
}
@@ -591,4 +628,12 @@ public class DistribPackageStore implements PackageStore {
}
return result;
}
-}
+
+ public static void deleteZKFileEntry(SolrZkClient client, String path) {
+ try {
+ client.delete(ZK_PACKAGESTORE + path, -1, true);
+ } catch (KeeperException | InterruptedException e) {
+ log.error("", e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
index db76e8a..921653f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
@@ -79,6 +79,13 @@ public interface PackageStore {
*/
void refresh(String path);
+ /** Delete a file cluster-wide */
+ void delete(String path);
+
+ /** Delete file from local file system */
+
+ void deleteLocal(String path);
+
public class FileEntry {
final ByteBuffer buf;
final MetaData meta;
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
index e71114e..db2e05e 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
@@ -132,6 +132,53 @@ public class PackageStoreAPI {
static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
@EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.DELETE,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public void delete(SolrQueryRequest req, SolrQueryResponse rsp) {
+ if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
+ throw new RuntimeException(PackageAPI.ERR_MSG);
+ }
+
+ try {
+ coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE, "true".getBytes(UTF_8),
+ CreateMode.EPHEMERAL, true);
+ String path = req.getPathTemplateValues().get("*");
+ validateName(path, true);
+ if(coreContainer.getPackageLoader().getPackageAPI().isJarInuse(path)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");
+ }
+ PackageStore.FileType type = packageStore.getType(path, true);
+ if(type == PackageStore.FileType.NOFILE) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + path);
+ }
+ packageStore.delete(path);
+ } catch (SolrException e){
+ throw e;
+ } catch (Exception e) {
+ log.error("Unknown error",e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } finally {
+ try {
+ coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);
+ } catch (Exception e) {
+ log.error("Unexpected error ", e);
+ }
+
+ }
+ }
+
+ @EndPoint(
+ path = "/node/files/*",
+ method = SolrRequest.METHOD.DELETE,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public void deleteLocal(SolrQueryRequest req, SolrQueryResponse rsp) {
+ String path = req.getPathTemplateValues().get("*");
+ validateName(path, true);
+ packageStore.deleteLocal(path);
+ }
+
+ @EndPoint(
path = "/cluster/files/*",
method = SolrRequest.METHOD.PUT,
permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index 5a3e29a..5a354ab 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -433,5 +433,22 @@ public class PackageAPI {
log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e));
}
-
+ public boolean isJarInuse(String path) {
+ Packages pkg = null;
+ try {
+ pkg = readPkgsFromZk(null, null);
+ } catch (KeeperException.NoNodeException nne) {
+ return false;
+ } catch (InterruptedException | KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ for (List<PkgVersion> vers : pkg.packages.values()) {
+ for (PkgVersion ver : vers) {
+ if (ver.files.contains(path)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
index ad39ad8..656624c 100644
--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
@@ -17,19 +17,9 @@
package org.apache.solr.filestore;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.function.Predicate;
-
import com.google.common.collect.ImmutableSet;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.client.methods.HttpDelete;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
@@ -51,6 +41,18 @@ import org.apache.zookeeper.server.ByteBufferInputStream;
import org.junit.After;
import org.junit.Before;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.function.Predicate;
+
import static org.apache.solr.common.util.Utils.JAVABINCONSUMER;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.hamcrest.CoreMatchers.containsString;
@@ -124,11 +126,8 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
Map expected = Utils.makeMap(
":files:/package/mypkg/v1.0/runtimelibs.jar:name", "runtimelibs.jar",
":files:/package/mypkg/v1.0[0]:sha512", "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"
-
);
- waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, true);
-
-
+ checkAllNodesForFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, true);
postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs_v2.jar.bin"),
"/package/mypkg/v1.0/runtimelibs_v2.jar",
null
@@ -137,11 +136,8 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
":files:/package/mypkg/v1.0/runtimelibs_v2.jar:name", "runtimelibs_v2.jar",
":files:/package/mypkg/v1.0[0]:sha512",
"bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417"
-
);
- waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs_v2.jar", expected, false);
-
-
+ checkAllNodesForFile(cluster,"/package/mypkg/v1.0/runtimelibs_v2.jar", expected, false);
expected = Utils.makeMap(
":files:/package/mypkg/v1.0", (Predicate<Object>) o -> {
@SuppressWarnings({"rawtypes"})
@@ -152,27 +148,31 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
for (Object file : l) {
if(! expectedKeys.contains(Utils.getObjectByPath(file, true, "name"))) return false;
}
-
return true;
}
);
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
String url = baseUrl + "/node/files/package/mypkg/v1.0?wt=javabin";
-
assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
-
}
-
-
-
+ // Delete Jars
+ DistribPackageStore.deleteZKFileEntry(cluster.getZkClient(), "/package/mypkg/v1.0/runtimelibs.jar");
+ JettySolrRunner j = cluster.getRandomJetty(random());
+ String path = j.getBaseURLV2() + "/cluster/files" + "/package/mypkg/v1.0/runtimelibs.jar";
+ HttpDelete del = new HttpDelete(path);
+ try(HttpSolrClient cl = (HttpSolrClient) j.newClient()) {
+ Utils.executeHttpMethod(cl.getHttpClient(), path, Utils.JSONCONSUMER, del);
+ }
+ expected = Collections.singletonMap(":files:/package/mypkg/v1.0/runtimelibs.jar", null);
+ checkAllNodesForFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, false);
} finally {
cluster.shutdown();
}
}
@SuppressWarnings({"unchecked", "rawtypes"})
- public static void waitForAllNodesHaveFile(MiniSolrCloudCluster cluster, String path, Map expected , boolean verifyContent) throws Exception {
+ public static void checkAllNodesForFile(MiniSolrCloudCluster cluster, String path, Map expected , boolean verifyContent) throws Exception {
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
String url = baseUrl + "/node/files" + path + "?wt=javabin&meta=true";
@@ -268,10 +268,9 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {
PackageUtils.uploadKey(bytes, path, Paths.get(jetty.getCoreContainer().getSolrHome()), client);
Object resp = Utils.executeGET(client.getHttpClient(), jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true", null);
- System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true"+" ,is: "+resp);
+ System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true" + " ,is: " + resp);
}
- waitForAllNodesHaveFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate<Object>) Objects::nonNull),
- false);
+ checkAllNodesForFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate<Object>) Objects::nonNull), false);
}
public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
@@ -298,9 +297,9 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
*/
public static byte[] readFile(String fname) throws IOException {
byte[] buf = null;
- try (FileInputStream fis = new FileInputStream(getFile(fname))) {
- buf = new byte[fis.available()];
- fis.read(buf);
+ try (InputStream is = TestDistribPackageStore.class.getClassLoader().getResourceAsStream(fname)) {
+ buf = new byte[is.available()];
+ is.read(buf);
}
return buf;
}
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 604332e..f967cbd 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -19,11 +19,9 @@ package org.apache.solr.pkg;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.Callable;
+
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
@@ -31,7 +29,11 @@ import org.apache.solr.client.solrj.*;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.*;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrResponseBase;
@@ -67,7 +69,9 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
-import static org.apache.solr.filestore.TestDistribPackageStore.*;
+import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
+import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
+import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
@@ -171,10 +175,10 @@ public class TestPackages extends SolrCloudTestCase {
V2Request v2r = new V2Request.Builder( "/c/"+COLLECTION_NAME+ "/config")
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload(plugins)
- .forceV2(true)
- .build();
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(plugins)
+ .forceV2(true)
+ .build();
cluster.getSolrClient().request(v2r);
verifyCmponent(cluster.getSolrClient(),
@@ -201,9 +205,9 @@ public class TestPackages extends SolrCloudTestCase {
cluster.getSolrClient() ,
new GenericSolrRequest(SolrRequest.METHOD.GET,
"/stream", new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
- WT, JAVABIN,
- "action", "plugins"
- ))), Utils.makeMap(
+ WT, JAVABIN,
+ "action", "plugins"
+ ))), Utils.makeMap(
":plugins:mincopy", "org.apache.solr.client.solrj.io.stream.metrics.MinCopyMetric"
));
@@ -412,10 +416,10 @@ public class TestPackages extends SolrCloudTestCase {
plugins.put("create-queryparser", p);
v2r = new V2Request.Builder( "/c/"+COLLECTION_NAME+ "/config")
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload(plugins)
- .forceV2(true)
- .build();
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(plugins)
+ .forceV2(true)
+ .build();
cluster.getSolrClient().request(v2r);
assertTrue(C.informCalled);
assertTrue(C2.informCalled);
@@ -428,14 +432,12 @@ public class TestPackages extends SolrCloudTestCase {
.setNode(jetty.getNodeName())
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 5);
- waitForAllNodesHaveFile(cluster,FILE3,
+ checkAllNodesForFile(cluster,FILE3,
Utils.makeMap(":files:" + FILE3 + ":name", "runtimelibs_v3.jar"),
false);
-
} finally {
cluster.shutdown();
}
-
}
@SuppressWarnings({"unchecked"})
private void executeReq(String uri, JettySolrRunner jetty, Utils.InputStreamConsumer parser, Map expected) throws Exception {
@@ -453,7 +455,7 @@ public class TestPackages extends SolrCloudTestCase {
}
private void verifyCmponent(SolrClient client, String COLLECTION_NAME,
- String componentType, String componentName, String pkg, String version) throws Exception {
+ String componentType, String componentName, String pkg, String version) throws Exception {
@SuppressWarnings({"unchecked"})
SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
WT, JAVABIN,
@@ -568,7 +570,7 @@ public class TestPackages extends SolrCloudTestCase {
delVersion.version = "0.12";//correct version. Should succeed
req.process(cluster.getSolrClient());
- //Verify with ZK that the data is correcy
+ //Verify with ZK that the data is correct
TestDistribPackageStore.assertResponseValues(1,
() -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
null, new Stat(), true))),
@@ -577,7 +579,6 @@ public class TestPackages extends SolrCloudTestCase {
":packages:test_pkg[0]:files[0]", FILE2
));
-
//So far we have been verifying the details with ZK directly
//use the package read API to verify with each node that it has the correct data
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
@@ -640,62 +641,62 @@ public class TestPackages extends SolrCloudTestCase {
System.setProperty("managed.schema.mutable", "true");
MiniSolrCloudCluster cluster =
- configureCluster(4)
- .withJettyConfig(jetty -> jetty.enableV2(true))
- .addConfig("conf", configset("cloud-managed"))
- .configure();
+ configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("cloud-managed"))
+ .configure();
try {
String FILE1 = "/schemapkg/schema-plugins.jar";
byte[] derFile = readFile("cryptokeys/pub_key512.der");
uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);
postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1,
- "iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");
+ "iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "schemapkg";
add.files = Arrays.asList(new String[]{FILE1});
V2Request req = new V2Request.Builder("/cluster/package")
- .forceV2(true)
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload(Collections.singletonMap("add", add))
- .build();
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
req.process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
- () -> new V2Request.Builder("/cluster/package").
- withMethod(SolrRequest.METHOD.GET)
- .build().process(cluster.getSolrClient()),
- Utils.makeMap(
- ":result:packages:schemapkg[0]:version", "1.0",
- ":result:packages:schemapkg[0]:files[0]", FILE1
- ));
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:schemapkg[0]:version", "1.0",
+ ":result:packages:schemapkg[0]:files[0]", FILE1
+ ));
CollectionAdminRequest
- .createCollection(COLLECTION_NAME, "conf", 2, 2)
- .process(cluster.getSolrClient());
+ .createCollection(COLLECTION_NAME, "conf", 2, 2)
+ .process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
String addFieldTypeAnalyzerWithClass = "{\n" +
- "'add-field-type' : {" +
- " 'name' : 'myNewTextFieldWithAnalyzerClass',\n" +
- " 'class':'schemapkg:my.pkg.MyTextField',\n" +
- " 'analyzer' : {\n" +
- " 'luceneMatchVersion':'5.0.0'" ;
+ "'add-field-type' : {" +
+ " 'name' : 'myNewTextFieldWithAnalyzerClass',\n" +
+ " 'class':'schemapkg:my.pkg.MyTextField',\n" +
+ " 'analyzer' : {\n" +
+ " 'luceneMatchVersion':'5.0.0'" ;
// + ",\n" +
// " 'class':'schemapkg:my.pkg.MyWhitespaceAnalyzer'\n";
String charFilters =
- " 'charFilters' : [{\n" +
- " 'class':'schemapkg:my.pkg.MyPatternReplaceCharFilterFactory',\n" +
- " 'replacement':'$1$1',\n" +
- " 'pattern':'([a-zA-Z])\\\\\\\\1+'\n" +
- " }],\n";
+ " 'charFilters' : [{\n" +
+ " 'class':'schemapkg:my.pkg.MyPatternReplaceCharFilterFactory',\n" +
+ " 'replacement':'$1$1',\n" +
+ " 'pattern':'([a-zA-Z])\\\\\\\\1+'\n" +
+ " }],\n";
String tokenizer =
- " 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";
+ " 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";
String filters =
- " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";
+ " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";
String suffix = " }\n" +
- "}}";
+ "}}";
cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {
@Override
@@ -719,34 +720,34 @@ public class TestPackages extends SolrCloudTestCase {
}
});
verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass",
- Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"1.0",
- ":fieldType:analyzer:tokenizer:_packageinfo_:version","1.0",
- ":fieldType:_packageinfo_:version","1.0"));
+ Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"1.0",
+ ":fieldType:analyzer:tokenizer:_packageinfo_:version","1.0",
+ ":fieldType:_packageinfo_:version","1.0"));
add = new Package.AddVersion();
add.version = "2.0";
add.pkg = "schemapkg";
add.files = Arrays.asList(new String[]{FILE1});
req = new V2Request.Builder("/cluster/package")
- .forceV2(true)
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload(Collections.singletonMap("add", add))
- .build();
+ .forceV2(true)
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(Collections.singletonMap("add", add))
+ .build();
req.process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
- () -> new V2Request.Builder("/cluster/package").
- withMethod(SolrRequest.METHOD.GET)
- .build().process(cluster.getSolrClient()),
- Utils.makeMap(
- ":result:packages:schemapkg[0]:version", "2.0",
- ":result:packages:schemapkg[0]:files[0]", FILE1
- ));
+ () -> new V2Request.Builder("/cluster/package").
+ withMethod(SolrRequest.METHOD.GET)
+ .build().process(cluster.getSolrClient()),
+ Utils.makeMap(
+ ":result:packages:schemapkg[0]:version", "2.0",
+ ":result:packages:schemapkg[0]:files[0]", FILE1
+ ));
verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass",
- Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"2.0",
- ":fieldType:analyzer:tokenizer:_packageinfo_:version","2.0",
- ":fieldType:_packageinfo_:version","2.0"));
+ Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"2.0",
+ ":fieldType:analyzer:tokenizer:_packageinfo_:version","2.0",
+ ":fieldType:_packageinfo_:version","2.0"));
} finally {
cluster.shutdown();
@@ -757,14 +758,14 @@ public class TestPackages extends SolrCloudTestCase {
private void verifySchemaComponent(SolrClient client, String COLLECTION_NAME, String path,
Map expected) throws Exception {
SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
- WT, JAVABIN,
- "meta", "true"));
+ WT, JAVABIN,
+ "meta", "true"));
GenericSolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET,path
- , params);
+ , params);
TestDistribPackageStore.assertResponseValues(10,
- client,
- req, expected);
+ client,
+ req, expected);
}
public static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception {
@@ -775,7 +776,7 @@ public class TestPackages extends SolrCloudTestCase {
fileContent,
path, sig);// has file, but no signature
- TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap(
+ TestDistribPackageStore.checkAllNodesForFile(cluster, path, Utils.makeMap(
":files:" + path + ":sha512",
sha512
), false);
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 498a827..46fcbdf 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -16,35 +16,6 @@
*/
package org.apache.solr.common.util;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.util.EntityUtils;
-import org.apache.solr.client.solrj.cloud.DistribStateManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
-import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.LinkedHashMapWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.MapWriterMap;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SpecProvider;
-import org.apache.solr.common.annotation.JsonProperty;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkOperation;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.server.ByteBufferInputStream;
-import org.noggit.CharArr;
-import org.noggit.JSONParser;
-import org.noggit.JSONWriter;
-import org.noggit.ObjectBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -88,6 +59,36 @@ import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.util.EntityUtils;
+import org.apache.solr.client.solrj.cloud.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
+import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.LinkedHashMapWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.MapWriterMap;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SpecProvider;
+import org.apache.solr.common.annotation.JsonProperty;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkOperation;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+import org.noggit.CharArr;
+import org.noggit.JSONParser;
+import org.noggit.JSONWriter;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
@@ -798,14 +799,17 @@ public class Utils {
public static <T> T executeGET(HttpClient client, String url, InputStreamConsumer<T> consumer) throws SolrException {
+ return executeHttpMethod(client, url, consumer, new HttpGet(url));
+ }
+
+ public static <T> T executeHttpMethod(HttpClient client, String url, InputStreamConsumer<T> consumer, HttpRequestBase httpMethod) {
T result = null;
- HttpGet httpGet = new HttpGet(url);
HttpResponse rsp = null;
try {
- rsp = client.execute(httpGet);
+ rsp = client.execute(httpMethod);
} catch (IOException e) {
log.error("Error in request to url : {}", url, e);
- throw new SolrException(SolrException.ErrorCode.UNKNOWN, "error sending request");
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, "Error sending request");
}
int statusCode = rsp.getStatusLine().getStatusCode();
if (statusCode != 200) {