You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/11/13 06:44:57 UTC
[lucene-solr] branch branch_8x updated: SOLR-13822: Added a
testcase and a bug fix
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new e428315 SOLR-13822: Added a testcase and a bug fix
e428315 is described below
commit e4283153878850b3e48be5da4913079c0492ff9d
Author: noble <no...@apache.org>
AuthorDate: Wed Nov 13 17:42:37 2019 +1100
SOLR-13822: Added a testcase and a bug fix
---
.../apache/solr/filestore/DistribPackageStore.java | 20 +++++++++++--------
.../src/java/org/apache/solr/pkg/PackageAPI.java | 23 +++++++++++++++++-----
.../java/org/apache/solr/pkg/PackageLoader.java | 7 +++++--
.../src/test/org/apache/solr/pkg/TestPackages.java | 12 +++++++++++
4 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index cd653e7..703102f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -71,15 +71,12 @@ public class DistribPackageStore implements PackageStore {
}
-
-
-
@Override
public Path getRealpath(String path) {
if (File.separatorChar == '\\') {
path = path.replace('/', File.separatorChar);
}
- if (path.charAt(0) != File.separatorChar) {
+ if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {
path = File.separator + path;
}
return new File(this.coreContainer.getResourceLoader().getInstancePath() +
@@ -222,7 +219,7 @@ public class DistribPackageStore implements PackageStore {
"?meta=true&wt=javabin&omitHeader=true";
boolean nodeHasBlob = false;
Object nl = Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), reqUrl, Utils.JAVABINCONSUMER);
- if (Utils.getObjectByPath(nl, false, Arrays.asList("files", getMetaPath())) != null) {
+ if (Utils.getObjectByPath(nl, false, Arrays.asList("files", path)) != null) {
nodeHasBlob = true;
}
@@ -292,7 +289,7 @@ public class DistribPackageStore implements PackageStore {
return;
}
ew.put("timestamp", getTimeStamp());
- if(metaData != null)
+ if (metaData != null)
metaData.writeMap(ew);
}
@@ -321,7 +318,7 @@ public class DistribPackageStore implements PackageStore {
byte[] bytes = baos.toByteArray();
info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length));
tmpFiles.put(entry.getPath(), info);
- List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
+ List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
int i = 0;
int FETCHFROM_SRC = 50;
String myNodeName = coreContainer.getZkController().getNodeName();
@@ -388,7 +385,14 @@ public class DistribPackageStore implements PackageStore {
}
if (from == null || "*".equals(from)) {
- f.fetchFromAnyNode();
+ log.info("Missing file in package store: {}", path);
+ if (f.fetchFromAnyNode()) {
+ log.info("Successfully downloaded : {}", path);
+ return true;
+ } else{
+ log.info("Unable to download file : {}", path);
+ return false;
+ }
} else {
f.fetchFileFromNodeAndPersist(from);
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index bf29b64..8327d36 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -18,6 +18,7 @@
package org.apache.solr.pkg;
import java.io.IOException;
+import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
@@ -35,7 +36,6 @@ import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ReflectMapWriter;
@@ -65,7 +65,7 @@ public class PackageAPI {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final CoreContainer coreContainer;
- private final ObjectMapper mapper = new ObjectMapper();
+ private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
private final PackageLoader packageLoader;
Packages pkgs;
@@ -75,10 +75,15 @@ public class PackageAPI {
public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
this.coreContainer = coreContainer;
this.packageLoader = loader;
- mapper.setAnnotationIntrospector(SolrJacksonAnnotationInspector.INSTANCE);
pkgs = new Packages();
SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
try {
+ pkgs = readPkgsFromZk(null, null);
+ } catch (KeeperException |InterruptedException e ) {
+ pkgs = new Packages();
+ //ignore
+ }
+ try {
registerListener(zkClient);
} catch (KeeperException | InterruptedException e) {
SolrZkClient.checkInterrupted(e);
@@ -129,7 +134,7 @@ public class PackageAPI {
if (data == null || stat == null) {
stat = new Stat();
data = coreContainer.getZkController().getZkClient()
- .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
+ .getData(SOLR_PKGS_PATH, null, stat, true);
}
Packages packages = null;
@@ -183,7 +188,6 @@ public class PackageAPI {
this.files = addVersion.files;
}
-
@Override
public boolean equals(Object obj) {
if (obj instanceof PkgVersion) {
@@ -194,6 +198,15 @@ public class PackageAPI {
}
return false;
}
+
+ @Override
+ public String toString() {
+ try {
+ return Utils.writeJson(this, new StringWriter(), false).toString() ;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
index 88e4e0f..565ff14 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
@@ -53,7 +53,7 @@ public class PackageLoader implements Closeable {
private final CoreContainer coreContainer;
private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
- private PackageAPI.Packages myCopy;
+ private PackageAPI.Packages myCopy = new PackageAPI.Packages();
private PackageAPI packageAPI;
@@ -61,7 +61,7 @@ public class PackageLoader implements Closeable {
public PackageLoader(CoreContainer coreContainer) {
this.coreContainer = coreContainer;
packageAPI = new PackageAPI(coreContainer, this);
- myCopy = packageAPI.pkgs;
+ refreshPackageConf();
}
@@ -104,6 +104,7 @@ public class PackageLoader implements Closeable {
for (SolrCore core : coreContainer.getCores()) {
core.getPackageListeners().packagesUpdated(updated);
}
+ myCopy = packageAPI.pkgs;
}
public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
@@ -248,6 +249,8 @@ public class PackageLoader implements Closeable {
this.version = v;
List<Path> paths = new ArrayList<>();
for (String file : version.files) {
+ //ensure that the files are downloaded and available
+ coreContainer.getPackageStoreAPI().getPackageStore().fetch(file,null);
paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
}
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 4ff0f7a..e5e9224 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -57,6 +57,7 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
+import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
@@ -297,6 +298,17 @@ public class TestPackages extends SolrCloudTestCase {
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "2.1" );
+ //we create a new node. This node does not have the packages. But it should download it from another node
+ JettySolrRunner jetty = cluster.startJettySolrRunner();
+ //create a new replica for this collection. it should end up
+ CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, "shard1")
+ .setNrtReplicas(1)
+ .setNode(jetty.getNodeName())
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 5);
+ waitForAllNodesHaveFile(cluster,FILE3,
+ Utils.makeMap(":files:" + FILE3 + ":name", "runtimelibs_v3.jar"),
+ false);
} finally {
cluster.shutdown();