You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/07/31 08:23:35 UTC
[lucene-solr] branch master updated: SOLR-14681: Introduce ability
to delete .jar stored in the Package Store (#1702)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new f443ede SOLR-14681: Introduce ability to delete .jar stored in the Package Store (#1702)
f443ede is described below
commit f443edebd207f93398b35520b24e2b66eba3609a
Author: Marcus <ma...@gmail.com>
AuthorDate: Fri Jul 31 01:23:18 2020 -0700
SOLR-14681: Introduce ability to delete .jar stored in the Package Store (#1702)
---
solr/CHANGES.txt | 2 +
.../apache/solr/filestore/DistribPackageStore.java | 67 ++++++++++++++++++----
.../org/apache/solr/filestore/PackageStore.java | 7 +++
.../org/apache/solr/filestore/PackageStoreAPI.java | 47 +++++++++++++++
.../src/java/org/apache/solr/pkg/PackageAPI.java | 19 +++++-
.../solr/filestore/TestDistribPackageStore.java | 38 ++++++------
.../src/test/org/apache/solr/pkg/TestPackages.java | 13 ++---
.../java/org/apache/solr/common/util/Utils.java | 10 +++-
8 files changed, 160 insertions(+), 43 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 028aedd..3050edf 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -21,6 +21,8 @@ New Features
* SOLR-14588: Introduce Circuit Breaker Infrastructure and a JVM heap usage memory tracking circuit breaker implementation (Atri Sharma)
+* SOLR-14681: Introduce ability to delete .jar stored in the Package Store. (MarcusSorealheis via Mike Drob)
+
Improvements
----------------------
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index 75165b4..021729f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -39,8 +39,11 @@ import java.util.function.Consumer;
import java.util.function.Predicate;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpDelete;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Utils;
@@ -90,7 +93,7 @@ public class DistribPackageStore implements PackageStore {
path = File.separator + path;
}
return new File(solrHome +
- File.separator + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath();
+ File.separator + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath();
}
class FileInfo {
@@ -182,8 +185,8 @@ public class DistribPackageStore implements PackageStore {
Map m = null;
try {
metadata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
- baseUrl + "/node/files" + getMetaPath(),
- Utils.newBytesConsumer((int) MAX_PKG_SIZE));
+ baseUrl + "/node/files" + getMetaPath(),
+ Utils.newBytesConsumer((int) MAX_PKG_SIZE));
m = (Map) Utils.fromJSON(metadata.array(), metadata.arrayOffset(), metadata.limit());
} catch (SolrException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching metadata", e);
@@ -191,8 +194,8 @@ public class DistribPackageStore implements PackageStore {
try {
ByteBuffer filedata = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
- baseUrl + "/node/files" + path,
- Utils.newBytesConsumer((int) MAX_PKG_SIZE));
+ baseUrl + "/node/files" + path,
+ Utils.newBytesConsumer((int) MAX_PKG_SIZE));
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(filedata));
String expected = (String) m.get("sha512");
if (!sha512.equals(expected)) {
@@ -216,7 +219,7 @@ public class DistribPackageStore implements PackageStore {
String baseurl = stateReader.getBaseUrlForNodeName(liveNode);
String url = baseurl.replace("/solr", "/api");
String reqUrl = url + "/node/files" + path +
- "?meta=true&wt=javabin&omitHeader=true";
+ "?meta=true&wt=javabin&omitHeader=true";
boolean nodeHasBlob = false;
Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER);
if (Utils.getObjectByPath(nl, false, Arrays.asList("files", path)) != null) {
@@ -331,7 +334,7 @@ public class DistribPackageStore implements PackageStore {
String dirName = info.path.substring(0, info.path.lastIndexOf('/'));
coreContainer.getZkController().getZkClient().makePath(ZK_PACKAGESTORE + dirName, false, true);
coreContainer.getZkController().getZkClient().create(ZK_PACKAGESTORE + info.path, info.getDetails().getMetaData().sha512.getBytes(UTF_8),
- CreateMode.PERSISTENT, true);
+ CreateMode.PERSISTENT, true);
} catch (Exception e) {
throw new SolrException(SERVER_ERROR, "Unable to create an entry in ZK", e);
}
@@ -369,7 +372,7 @@ public class DistribPackageStore implements PackageStore {
//fire and forget
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), url, null);
} catch (Exception e) {
- log.info("Node: {} failed to respond for file fetch notification", node, e);
+ log.info("Node: {} failed to respond for file fetch notification", node, e);
//ignore the exception
// some nodes may be down or not responding
}
@@ -471,12 +474,46 @@ public class DistribPackageStore implements PackageStore {
}
@Override
+ public void delete(String path) {
+ deleteLocal(path);
+ List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
+ HttpClient client = coreContainer.getUpdateShardHandler().getDefaultHttpClient();
+ for (String node : nodes) {
+ String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node);
+ String url = baseUrl.replace("/solr", "/api") + "/node/files" + path;
+ HttpDelete del = new HttpDelete(url);
+ coreContainer.runAsync(() -> Utils.executeHttpMethod(client, url, null, del));//invoke delete command on all nodes asynchronously
+ }
+ }
+
+ private void checkInZk(String path) {
+ try {
+ //fail if file exists
+ if (coreContainer.getZkController().getZkClient().exists(ZK_PACKAGESTORE + path, true)) {
+ throw new SolrException(BAD_REQUEST, "The path exist ZK, delete and retry");
+ }
+
+ } catch (SolrException se) {
+ throw se;
+ } catch (Exception e) {
+ log.error("Could not connect to ZK", e);
+ }
+ }
+
+ @Override
+ public void deleteLocal(String path) {
+ checkInZk(path);
+ FileInfo f = new FileInfo(path);
+ f.deleteFile();
+ }
+
+ @Override
public void refresh(String path) {
try {
@SuppressWarnings({"rawtypes"})
List l = null;
try {
- l = coreContainer.getZkController().getZkClient().getChildren(ZK_PACKAGESTORE+ path, null, true);
+ l = coreContainer.getZkController().getZkClient().getChildren(ZK_PACKAGESTORE + path, null, true);
} catch (KeeperException.NoNodeException e) {
// does not matter
}
@@ -485,7 +522,7 @@ public class DistribPackageStore implements PackageStore {
List myFiles = list(path, s -> true);
for (Object f : l) {
if (!myFiles.contains(f)) {
- log.info("{} does not exist locally, downloading.. ",f);
+ log.info("{} does not exist locally, downloading.. ", f);
fetch(path + "/" + f.toString(), "*");
}
}
@@ -591,4 +628,12 @@ public class DistribPackageStore implements PackageStore {
}
return result;
}
-}
+
+ public static void deleteZKFileEntry(SolrZkClient client, String path) {
+ try {
+ client.delete(ZK_PACKAGESTORE + path, -1, true);
+ } catch (KeeperException | InterruptedException e) {
+ log.error("", e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
index db76e8a..921653f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java
@@ -79,6 +79,13 @@ public interface PackageStore {
*/
void refresh(String path);
+ /** Delete a file cluster-wide */
+ void delete(String path);
+
+ /** Delete file from local file system */
+
+ void deleteLocal(String path);
+
public class FileEntry {
final ByteBuffer buf;
final MetaData meta;
diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
index e71114e..db2e05e 100644
--- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
+++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
@@ -132,6 +132,53 @@ public class PackageStoreAPI {
static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
@EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.DELETE,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public void delete(SolrQueryRequest req, SolrQueryResponse rsp) {
+ if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
+ throw new RuntimeException(PackageAPI.ERR_MSG);
+ }
+
+ try {
+ coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE, "true".getBytes(UTF_8),
+ CreateMode.EPHEMERAL, true);
+ String path = req.getPathTemplateValues().get("*");
+ validateName(path, true);
+ if(coreContainer.getPackageLoader().getPackageAPI().isJarInuse(path)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");
+ }
+ PackageStore.FileType type = packageStore.getType(path, true);
+ if(type == PackageStore.FileType.NOFILE) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + path);
+ }
+ packageStore.delete(path);
+ } catch (SolrException e){
+ throw e;
+ } catch (Exception e) {
+ log.error("Unknown error",e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } finally {
+ try {
+ coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);
+ } catch (Exception e) {
+ log.error("Unexpected error ", e);
+ }
+
+ }
+ }
+
+ @EndPoint(
+ path = "/node/files/*",
+ method = SolrRequest.METHOD.DELETE,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public void deleteLocal(SolrQueryRequest req, SolrQueryResponse rsp) {
+ String path = req.getPathTemplateValues().get("*");
+ validateName(path, true);
+ packageStore.deleteLocal(path);
+ }
+
+ @EndPoint(
path = "/cluster/files/*",
method = SolrRequest.METHOD.PUT,
permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index 5a3e29a..5a354ab 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -433,5 +433,22 @@ public class PackageAPI {
log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e));
}
-
+ public boolean isJarInuse(String path) {
+ Packages pkg = null;
+ try {
+ pkg = readPkgsFromZk(null, null);
+ } catch (KeeperException.NoNodeException nne) {
+ return false;
+ } catch (InterruptedException | KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ for (List<PkgVersion> vers : pkg.packages.values()) {
+ for (PkgVersion ver : vers) {
+ if (ver.files.contains(path)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
index 1d7480a..f881156 100644
--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
@@ -19,6 +19,7 @@ package org.apache.solr.filestore;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.http.client.methods.HttpDelete;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
@@ -39,15 +40,15 @@ import org.apache.solr.util.LogLevel;
import org.apache.zookeeper.server.ByteBufferInputStream;
import org.junit.After;
import org.junit.Before;
-
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
-import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.Collections;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
@@ -124,11 +125,8 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
Map expected = Utils.makeMap(
":files:/package/mypkg/v1.0/runtimelibs.jar:name", "runtimelibs.jar",
":files:/package/mypkg/v1.0[0]:sha512", "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"
-
);
- waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, true);
-
-
+ checkAllNodesForFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, true);
postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs_v2.jar.bin"),
"/package/mypkg/v1.0/runtimelibs_v2.jar",
null
@@ -137,11 +135,8 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
":files:/package/mypkg/v1.0/runtimelibs_v2.jar:name", "runtimelibs_v2.jar",
":files:/package/mypkg/v1.0[0]:sha512",
"bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417"
-
);
- waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs_v2.jar", expected, false);
-
-
+ checkAllNodesForFile(cluster,"/package/mypkg/v1.0/runtimelibs_v2.jar", expected, false);
expected = Utils.makeMap(
":files:/package/mypkg/v1.0", (Predicate<Object>) o -> {
@SuppressWarnings({"rawtypes"})
@@ -152,27 +147,31 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
for (Object file : l) {
if(! expectedKeys.contains(Utils.getObjectByPath(file, true, "name"))) return false;
}
-
return true;
}
);
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
String url = baseUrl + "/node/files/package/mypkg/v1.0?wt=javabin";
-
assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
-
}
-
-
-
+ // Delete Jars
+ DistribPackageStore.deleteZKFileEntry(cluster.getZkClient(), "/package/mypkg/v1.0/runtimelibs.jar");
+ JettySolrRunner j = cluster.getRandomJetty(random());
+ String path = j.getBaseURLV2() + "/cluster/files" + "/package/mypkg/v1.0/runtimelibs.jar";
+ HttpDelete del = new HttpDelete(path);
+ try(HttpSolrClient cl = (HttpSolrClient) j.newClient()) {
+ Utils.executeHttpMethod(cl.getHttpClient(), path, Utils.JSONCONSUMER, del);
+ }
+ expected = Collections.singletonMap(":files:/package/mypkg/v1.0/runtimelibs.jar", null);
+ checkAllNodesForFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, false);
} finally {
cluster.shutdown();
}
}
@SuppressWarnings({"unchecked", "rawtypes"})
- public static void waitForAllNodesHaveFile(MiniSolrCloudCluster cluster, String path, Map expected , boolean verifyContent) throws Exception {
+ public static void checkAllNodesForFile(MiniSolrCloudCluster cluster, String path, Map expected , boolean verifyContent) throws Exception {
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api");
String url = baseUrl + "/node/files" + path + "?wt=javabin&meta=true";
@@ -268,10 +267,9 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {
PackageUtils.uploadKey(bytes, path, Paths.get(jetty.getCoreContainer().getSolrHome()), client);
Object resp = Utils.executeGET(client.getHttpClient(), jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true", null);
- System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true"+" ,is: "+resp);
+ System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true" + " ,is: " + resp);
}
- waitForAllNodesHaveFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate<Object>) Objects::nonNull),
- false);
+ checkAllNodesForFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate<Object>) Objects::nonNull), false);
}
public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index a54a3b0..2257cf3 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -71,7 +71,7 @@ import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
-import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile;
+import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
@@ -432,14 +432,12 @@ public class TestPackages extends SolrCloudTestCase {
.setNode(jetty.getNodeName())
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 5);
- waitForAllNodesHaveFile(cluster,FILE3,
+ checkAllNodesForFile(cluster,FILE3,
Utils.makeMap(":files:" + FILE3 + ":name", "runtimelibs_v3.jar"),
false);
-
} finally {
cluster.shutdown();
}
-
}
@SuppressWarnings({"unchecked"})
@@ -575,7 +573,7 @@ public class TestPackages extends SolrCloudTestCase {
delVersion.version = "0.12";//correct version. Should succeed
req.process(cluster.getSolrClient());
- //Verify with ZK that the data is correcy
+ //Verify with ZK that the data is correct
TestDistribPackageStore.assertResponseValues(1,
() -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
null, new Stat(), true))),
@@ -584,7 +582,6 @@ public class TestPackages extends SolrCloudTestCase {
":packages:test_pkg[0]:files[0]", FILE2
));
-
//So far we have been verifying the details with ZK directly
//use the package read API to verify with each node that it has the correct data
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
@@ -782,8 +779,8 @@ public class TestPackages extends SolrCloudTestCase {
fileContent,
path, sig);// has file, but no signature
- TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap(
- ":files:" + path + ":sha512",
+ TestDistribPackageStore.checkAllNodesForFile(cluster, path, Utils.makeMap(
+ ":files:" + path + ":sha512",
sha512
), false);
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 3310480..2e68b19 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -62,6 +62,7 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.VersionedData;
@@ -793,14 +794,17 @@ public class Utils {
public static <T> T executeGET(HttpClient client, String url, InputStreamConsumer<T> consumer) throws SolrException {
+ return executeHttpMethod(client, url, consumer, new HttpGet(url));
+ }
+
+ public static <T> T executeHttpMethod(HttpClient client, String url, InputStreamConsumer<T> consumer, HttpRequestBase httpMethod) {
T result = null;
- HttpGet httpGet = new HttpGet(url);
HttpResponse rsp = null;
try {
- rsp = client.execute(httpGet);
+ rsp = client.execute(httpMethod);
} catch (IOException e) {
log.error("Error in request to url : {}", url, e);
- throw new SolrException(SolrException.ErrorCode.UNKNOWN, "error sending request");
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, "Error sending request");
}
int statusCode = rsp.getStatusLine().getStatusCode();
if (statusCode != 200) {