You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/12/18 19:24:49 UTC
[lucene-solr] branch branch_8x updated: SOLR-13822: Bug fixs and
tests for URP loading
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new ac7ad67 SOLR-13822: Bug fixs and tests for URP loading
ac7ad67 is described below
commit ac7ad676ef230dc66c73aeeddc0224ff05a43d2a
Author: noble <no...@apache.org>
AuthorDate: Thu Nov 21 11:25:16 2019 +1100
SOLR-13822: Bug fixs and tests for URP loading
---
.../src/java/org/apache/solr/core/PluginBag.java | 5 +--
.../apache/solr/filestore/DistribPackageStore.java | 2 +-
.../src/java/org/apache/solr/pkg/PackageAPI.java | 14 +++++-
.../processor/UpdateRequestProcessorChain.java | 8 +++-
.../test-files/runtimecode/TestVersionedURP.java | 29 ++++++++++++
solr/core/src/test-files/runtimecode/sig.txt | 15 +++++++
.../src/test-files/runtimecode/testurp_v1.jar.bin | Bin 0 -> 689 bytes
.../src/test-files/runtimecode/testurp_v2.jar.bin | Bin 0 -> 689 bytes
.../src/test/org/apache/solr/pkg/TestPackages.java | 50 +++++++++++++++++++--
.../src/package-manager-internals.adoc | 12 ++---
10 files changed, 117 insertions(+), 18 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index f547d10..b26b987 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -47,7 +47,6 @@ import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
-import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.CryptoKeys;
import org.apache.solr.util.SimplePostTool;
@@ -143,9 +142,7 @@ public class PluginBag<T> implements AutoCloseable {
} else {
if (info.pkgName != null) {
PackagePluginHolder<T> holder = new PackagePluginHolder<>(info, core, meta);
- return meta.clazz == UpdateRequestProcessorFactory.class ?
- new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) :
- holder;
+ return holder;
} else {
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName));
initInstance(inst, info);
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index 3e3b583..8d9af8f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -459,8 +459,8 @@ public class DistribPackageStore implements PackageStore {
if (fetch(path, null)) {
file = getRealpath(path).toFile();
}
- if (!file.exists()) return FileType.NOFILE;
}
+ if (!file.exists()) return FileType.NOFILE;
if (file.isDirectory()) return FileType.DIRECTORY;
return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE;
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index d80e8bd..1b93d89 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -271,8 +271,18 @@ public class PackageAPI {
log.error("Error deserializing packages.json", e);
packages = new Packages();
}
- packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
- packages.znodeVersion = stat.getVersion() ;
+ List list = packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN);
+ for (Object o : list) {
+ if (o instanceof PkgVersion) {
+ PkgVersion version = (PkgVersion) o;
+ if (Objects.equals(version.version, add.version)) {
+ payload.addError("Version '" + add.version + "' exists already");
+ return null;
+ }
+ }
+ }
+ list.add(new PkgVersion(add));
+ packages.znodeVersion = stat.getVersion() + 1;
finalState[0] = packages;
return Utils.toJSON(packages);
});
diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
index 5ddaf8b..a912e2a 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
@@ -296,7 +296,13 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
for (String s : names) {
s = s.trim();
if (s.isEmpty()) continue;
- UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s);
+ UpdateRequestProcessorFactory p = null;
+ PluginBag.PluginHolder<UpdateRequestProcessorFactory> holder = core.getUpdateProcessors().getRegistry().get(s);
+ if (holder instanceof PackagePluginHolder) {
+ p = new LazyUpdateRequestProcessorFactory(holder);
+ } else {
+ p = core.getUpdateProcessors().get(s);
+ }
if (p == null) {
Class<UpdateRequestProcessorFactory> factoryClass = implicits.get(s);
if(factoryClass != null) {
diff --git a/solr/core/src/test-files/runtimecode/TestVersionedURP.java b/solr/core/src/test-files/runtimecode/TestVersionedURP.java
new file mode 100644
index 0000000..ae91b80
--- /dev/null
+++ b/solr/core/src/test-files/runtimecode/TestVersionedURP.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package runtimecode;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.SimpleUpdateProcessorFactory;
+
+public class TestVersionedURP extends SimpleUpdateProcessorFactory {
+ @Override
+ protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) {
+ cmd.solrDoc.addField("TestVersionedURP.Ver_s", "Version 2");
+ }
+}
diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt
index 4ef8e9c..f354b5f 100644
--- a/solr/core/src/test-files/runtimecode/sig.txt
+++ b/solr/core/src/test-files/runtimecode/sig.txt
@@ -57,6 +57,14 @@ openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem cache_v2.jar.bin | openss
SOrekHt+uup+z2z+nZU5indk2huRRfmbM+W+vQ0variHrcZEG9EXt5LuPFl8Ki9A
hr6klMHdVP8nj4wuQhu/Hg==
+openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v1.jar.bin | openssl enc -base64 | tr -d \\n | sed
+
+h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ==
+
+openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v2.jar.bin | openssl enc -base64 | openssl enc -base64 | tr -d \\n | sed
+
+P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw==
+
====================sha512====================
openssl dgst -sha512 runtimelibs.jar.bin
@@ -80,6 +88,13 @@ openssl dgst -sha512 cache_v2.jar.bin
873337e67b90b7ff99df012b2e9093c63079c37a564643d34861a88c4cbaf0698ebb096905929d69cdbde3b4d29d55e31db24ee05c01b39c0b75a16e54eb4335
+openssl dgst -sha512 testurp_v1.jar.bin
+
+7b9df184202e62cc6f73e69a7f369a6f469ad9e1508a2d61eeb7744b204f2489a2c617808f28a496fcbca3a318e16dca238c111e60a903ace877a79900bb8729
+
+openssl dgst -sha512 testurp_v2.jar.bin
+
+5c4c0c454a032916e48a1c14a0fecbd6658658a66aedec5168b7222f2e3c0c63fbe09637238a9325ce2e95a2c8521834397a97701ead46c681aa20c9fccb6654
=============sha256============================
openssl dgst -sha256 runtimelibs.jar.bin
diff --git a/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin b/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin
new file mode 100644
index 0000000..7c9044c
Binary files /dev/null and b/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin differ
diff --git a/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin b/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin
new file mode 100644
index 0000000..0c8041e
Binary files /dev/null and b/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin differ
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 560cdea..baa8a99 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -26,6 +26,7 @@ import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -34,14 +35,17 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.request.beans.Package;
+import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.ConfigRequest;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.MapWriterMap;
import org.apache.solr.common.NavigableObject;
+import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -86,6 +90,8 @@ public class TestPackages extends SolrCloudTestCase {
String FILE1 = "/mypkg/runtimelibs.jar";
String FILE2 = "/mypkg/runtimelibs_v2.jar";
String FILE3 = "/mypkg/runtimelibs_v3.jar";
+ String URP1 = "/mypkg/testurpv1.jar";
+ String URP2 = "/mypkg/testurpv2.jar";
String COLLECTION_NAME = "testPluginLoadingColl";
byte[] derFile = readFile("cryptokeys/pub_key512.der");
cluster.getZkClient().makePath("/keys/exe", true);
@@ -93,10 +99,13 @@ public class TestPackages extends SolrCloudTestCase {
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
"L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+ postFileAndWait(cluster, "runtimecode/testurp_v1.jar.bin", URP1,
+ "h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ==");
+
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "mypkg";
- add.files = Arrays.asList(new String[]{FILE1});
+ add.files = Arrays.asList(new String[]{FILE1, URP1});
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
@@ -125,6 +134,7 @@ public class TestPackages extends SolrCloudTestCase {
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," +
"'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," +
"'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" +
+ "'create-updateProcessor' : { 'name' : 'myurp', 'class': 'mypkg:org.apache.solr.update.TestVersionedURP' }" +
"}";
cluster.getSolrClient().request(new ConfigRequest(payload) {
@Override
@@ -145,7 +155,20 @@ public class TestPackages extends SolrCloudTestCase {
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "1.0" );
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "updateProcessor", "myurp",
+ "mypkg", "1.0" );
+
+ UpdateRequest ur = new UpdateRequest();
+ ur.add(new SolrInputDocument("id", "1"));
+ ur.setParam("processor", "myurp");
+ ur.process(cluster.getSolrClient(), COLLECTION_NAME);
+ cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
+
+ QueryResponse result = cluster.getSolrClient()
+ .query(COLLECTION_NAME, new SolrQuery( "id:1"));
+ assertEquals("Version 1", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s"));
executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
Utils.JAVABINCONSUMER,
@@ -165,9 +188,11 @@ public class TestPackages extends SolrCloudTestCase {
postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
"j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+ postFileAndWait(cluster, "runtimecode/testurp_v2.jar.bin", URP2,
+ "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw==");
//add the version using package API
add.version = "1.1";
- add.files = Arrays.asList(new String[]{FILE2});
+ add.files = Arrays.asList(new String[]{FILE2,URP2});
req.process(cluster.getSolrClient());
verifyCmponent(cluster.getSolrClient(),
@@ -182,6 +207,11 @@ public class TestPackages extends SolrCloudTestCase {
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "1.1" );
+ verifyCmponent(cluster.getSolrClient(),
+ COLLECTION_NAME, "updateProcessor", "myurp",
+ "mypkg", "1.1" );
+
+
executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
Utils.JSONCONSUMER,
Utils.makeMap( "Version","2"));
@@ -192,7 +222,7 @@ public class TestPackages extends SolrCloudTestCase {
"a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==");
add.version = "2.1";
- add.files = Arrays.asList(new String[]{FILE3});
+ add.files = Arrays.asList(new String[]{FILE3, URP2});
req.process(cluster.getSolrClient());
//now let's verify that the classes are updated
@@ -212,6 +242,18 @@ public class TestPackages extends SolrCloudTestCase {
Utils.JSONCONSUMER,
Utils.makeMap("Version","2"));
+ //insert a doc with urp
+ ur = new UpdateRequest();
+ ur.add(new SolrInputDocument("id", "2"));
+ ur.setParam("processor", "myurp");
+ ur.process(cluster.getSolrClient(), COLLECTION_NAME);
+ cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
+
+ result = cluster.getSolrClient()
+ .query(COLLECTION_NAME, new SolrQuery( "id:2"));
+
+ assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s"));
+
Package.DelVersion delVersion = new Package.DelVersion();
delVersion.pkg = "mypkg";
@@ -262,7 +304,7 @@ public class TestPackages extends SolrCloudTestCase {
}.process(cluster.getSolrClient()) ;
add.version = "2.1";
- add.files = Arrays.asList(new String[]{FILE3});
+ add.files = Arrays.asList(new String[]{FILE3, URP2});
req.process(cluster.getSolrClient());
//the collections mypkg is set to use version 1.1
diff --git a/solr/solr-ref-guide/src/package-manager-internals.adoc b/solr/solr-ref-guide/src/package-manager-internals.adoc
index 90ed865..6cb1956 100644
--- a/solr/solr-ref-guide/src/package-manager-internals.adoc
+++ b/solr/solr-ref-guide/src/package-manager-internals.adoc
@@ -1,7 +1,7 @@
= Package Manager Internals
The package manager (CLI) internally uses various Solr APIs to install, deploy and update packages. This document contains an overview of those APIs.
-== Design Objectives
+== Salient Features
* Zero disruption deployment (hot deployment): Should be possible to install and update packages without node restarts or core reloads, and hence deployments should be quick and without failed requests or stale caches.
* Easy packaging:
@@ -81,13 +81,13 @@ $ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/maste
2) Sign the jar with your private key
[source, bash]
----
-$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
+$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed
----
-3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command) . Ensure that new lines and spaces are removed. Do not forget to do URL encoding of you signature( e.g escape `+` with `%2B`)
+3) Upload your jar with signature. (replace the `sig` param with the output you got from the previous command)
[source, bash]
----
-$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ==
+$ curl --data-binary @runtimelibs.jar -X PUT http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
----
4) Verify your jar upload
@@ -263,8 +263,8 @@ Get a new version of the jar, sign and upload it
[source, bash]
----
$ curl -o runtimelibs3.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true
-$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
-$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ==
+$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed
+$ curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=<signature>
----
7) Verify it