You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/06/11 19:32:52 UTC
[lucene-solr] branch jira/SOLR-13534 updated: SOLR-13534: Dynamic
loading of jars from a remote url
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/SOLR-13534
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-13534 by this push:
new 21cc383 SOLR-13534: Dynamic loading of jars from a remote url
21cc383 is described below
commit 21cc3835d951d200fe2c2a525fc4b41f449119ab
Author: Noble Paul <no...@apache.org>
AuthorDate: Wed Jun 12 05:32:28 2019 +1000
SOLR-13534: Dynamic loading of jars from a remote url
---
.../java/org/apache/solr/core/BlobRepository.java | 63 ++++++++-
.../java/org/apache/solr/core/ConfigOverlay.java | 6 +-
.../java/org/apache/solr/core/MemClassLoader.java | 20 ++-
.../src/java/org/apache/solr/core/PluginBag.java | 47 +++++--
.../src/java/org/apache/solr/core/SolrCore.java | 3 +
.../org/apache/solr/handler/SolrConfigHandler.java | 10 ++
.../solr/core/BlobRepositoryMockingTest.java | 44 +++++-
.../apache/solr/core/TestDynamicLoadingUrl.java | 153 +++++++++++++++++++++
.../apache/solr/core/TestSolrConfigHandler.java | 3 +-
.../adding-custom-plugins-in-solrcloud-mode.adoc | 77 ++++++++++-
10 files changed, 405 insertions(+), 21 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
index 0cf6dfb..4d31c31 100644
--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
@@ -16,12 +16,13 @@
*/
package org.apache.solr.core;
-import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE;
-import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
-
+import java.io.FileInputStream;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -44,12 +45,17 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.SimplePostTool;
import org.apache.zookeeper.server.ByteBufferInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE;
+import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
+
/**
* The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the Jar in a single node.
*/
@@ -108,6 +114,14 @@ public class BlobRepository {
return getBlobIncRef(key.concat(decoder.getName()), () -> addBlob(key,decoder));
}
+ BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha512) {
+ StringBuffer keyBuilder = new StringBuffer(key);
+ if (decoder != null) keyBuilder .append( decoder.getName());
+ keyBuilder.append("/").append( sha512);
+
+ return getBlobIncRef(keyBuilder.toString(), () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha512), decoder));
+ }
+
// do the actual work returning the appropriate type...
private <T> BlobContentRef<T> getBlobIncRef(String key, Callable<BlobContent<T>> blobCreator) {
BlobContent<T> aBlob;
@@ -149,14 +163,41 @@ public class BlobRepository {
blobs.put(keyPlusName, aBlob);
return aBlob;
}
-
+ static String INVALID_JAR_MSG = "Invalid jar from {0} , expected sha512 hash : {1} , actual : {2}";
+
+ private ByteBuffer fetchBlobAndVerify(String key, String url, String sha512) {
+ ByteBuffer byteBuffer = fetchFromUrl(key, url);
+ String computedDigest = sha512Digest(byteBuffer);
+ if (!computedDigest.equals(sha512)) {
+ throw new SolrException(SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha512, computedDigest));
+
+ }
+ return byteBuffer;
+ }
+
+ public static String sha512Digest(ByteBuffer byteBuffer) {
+ MessageDigest digest = null;
+ try {
+ digest = MessageDigest.getInstance("SHA-512");
+ } catch (NoSuchAlgorithmException e) {
+ //unlikely
+ throw new SolrException(SERVER_ERROR, e);
+ }
+ digest.update(byteBuffer);
+ return String.format("%0128x", new BigInteger(1, digest.digest()));
+ }
+
+
/**
* Package local for unit tests only please do not use elsewhere
*/
ByteBuffer fetchBlob(String key) {
Replica replica = getSystemCollReplica();
String url = replica.getStr(BASE_URL_PROP) + "/" + CollectionAdminParams.SYSTEM_COLL + "/blob/" + key + "?wt=filestream";
+ return fetchFromUrl(key, url);
+ }
+ ByteBuffer fetchFromUrl(String key, String url) {
HttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient();
HttpGet httpGet = new HttpGet(url);
ByteBuffer b;
@@ -240,7 +281,7 @@ public class BlobRepository {
public BlobContent(String key, ByteBuffer buffer, Decoder<T> decoder) {
this.key = key;
- this.content = decoder.decode(new ByteBufferInputStream(buffer));
+ this.content = decoder == null ? (T) buffer : decoder.decode(new ByteBufferInputStream(buffer));
}
@SuppressWarnings("unchecked")
@@ -288,4 +329,16 @@ public class BlobRepository {
}
}
+ public static void main(String[] args) throws Exception {
+
+
+ try (InputStream in = new FileInputStream("/Users/noble/work/2-lucene-solr-github/solr/dist/solr-analysis-extras-9.0.0-SNAPSHOT.jar")) {
+ byte[] b = new byte[32 * 1024];
+ int sz = in.read(b);
+ MessageDigest digest = MessageDigest.getInstance("SHA-512");
+ digest.update(b, 0, sz);
+ String computedDigest = String.format("%0128x", new BigInteger(1, digest.digest()));
+ System.out.println(computedDigest);
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java b/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
index 2852ba5..967db85 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
@@ -238,9 +238,9 @@ public class ConfigOverlay implements MapSerializable {
public ConfigOverlay addNamedPlugin(Map<String, Object> info, String typ) {
Map dataCopy = Utils.getDeepCopy(data, 4);
- Map reqHandler = (Map) dataCopy.get(typ);
- if (reqHandler == null) dataCopy.put(typ, reqHandler = new LinkedHashMap());
- reqHandler.put(info.get(CoreAdminParams.NAME), info);
+ Map existing = (Map) dataCopy.get(typ);
+ if (existing == null) dataCopy.put(typ, existing = new LinkedHashMap());
+ existing.put(info.get(CoreAdminParams.NAME), info);
return new ConfigOverlay(dataCopy, this.znodeVersion);
}
diff --git a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
index a249dc8..372be78 100644
--- a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.StrUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +45,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
private final SolrResourceLoader parentLoader;
private List<PluginBag.RuntimeLib> libs = new ArrayList<>();
private Map<String, Class> classCache = new HashMap<>();
+ private List<String> errors = new ArrayList<>();
public MemClassLoader(List<PluginBag.RuntimeLib> libs, SolrResourceLoader resourceLoader) {
@@ -51,6 +53,21 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
this.libs = libs;
}
+ synchronized void loadRemoteJars() {
+ if (allJarsLoaded) return;
+ int count = 0;
+ for (PluginBag.RuntimeLib lib : libs) {
+ if (lib.getUrl() != null) {
+ try {
+ lib.loadJar();
+ } catch (Exception e) {
+ log.error("Error loading runtime library", e);
+ }
+ count++;
+ }
+ }
+ if (count == libs.size()) allJarsLoaded = true;
+ }
public synchronized void loadJars() {
if (allJarsLoaded) return;
@@ -60,6 +77,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
lib.loadJar();
lib.verify();
} catch (Exception exception) {
+ errors.add(exception.getMessage());
if (exception instanceof SolrException) throw (SolrException) exception;
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Atleast one runtimeLib could not be loaded", exception);
}
@@ -87,7 +105,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
try {
buf = getByteBuffer(name, jarName);
} catch (Exception e) {
- throw new ClassNotFoundException("class could not be loaded " + name, e);
+ throw new ClassNotFoundException("class could not be loaded " + name + (errors.isEmpty()? "": "Some dynamic libraries could not be loaded: "+ StrUtils.join(errors, '|')), e);
}
if (buf == null) throw new ClassNotFoundException("Class not found :" + name);
ProtectionDomain defaultDomain = null;
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index abd1a44..bd73074 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -464,7 +464,7 @@ public class PluginBag<T> implements AutoCloseable {
* This represents a Runtime Jar. A jar requires two details , name and version
*/
public static class RuntimeLib implements PluginInfoInitialized, AutoCloseable {
- private String name, version, sig;
+ private String name, version, sig, sha512, url;
private BlobRepository.BlobContentRef<ByteBuffer> jarContent;
private final CoreContainer coreContainer;
private boolean verified = false;
@@ -472,24 +472,49 @@ public class PluginBag<T> implements AutoCloseable {
@Override
public void init(PluginInfo info) {
name = info.attributes.get(NAME);
- Object v = info.attributes.get("version");
- if (name == null || v == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "runtimeLib must have name and version");
+ url = info.attributes.get("url");
+ if(url == null) {
+ Object v = info.attributes.get("version");
+ if (name == null || v == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "runtimeLib must have name and version");
+ }
+ version = String.valueOf(v);
+ sig = info.attributes.get("sig");
+ } else {
+ sha512 = info.attributes.get("sha512");
+ if(sha512 == null){
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute");
+ }
+ ByteBuffer buf = null;
+ buf = coreContainer.getBlobRepository().fetchFromUrl(name, url);
+
+ String digest = BlobRepository.sha512Digest(buf);
+ if(!sha512.equals(digest)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest) );
+ }
+ log.info("dynamic library verified {}, sha512: {}", url, sha512);
+
}
- version = String.valueOf(v);
- sig = info.attributes.get("sig");
+
}
public RuntimeLib(SolrCore core) {
coreContainer = core.getCoreContainer();
}
+ public String getUrl(){
+ return url;
+ }
void loadJar() {
if (jarContent != null) return;
synchronized (this) {
if (jarContent != null) return;
- jarContent = coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version);
+
+ jarContent = url == null?
+ coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version):
+ coreContainer.getBlobRepository().getBlobIncRef(name, null,url,sha512);
+
}
}
@@ -508,7 +533,7 @@ public class PluginBag<T> implements AutoCloseable {
public ByteBuffer getFileContent(String entryName) throws IOException {
if (jarContent == null)
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name + "/" + version);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name );
return getFileContent(jarContent.blob, entryName);
}
@@ -547,7 +572,11 @@ public class PluginBag<T> implements AutoCloseable {
List<RuntimeLib> l = new ArrayList<>(libs.size());
for (PluginInfo lib : libs) {
RuntimeLib rtl = new RuntimeLib(core);
- rtl.init(lib);
+ try {
+ rtl.init(lib);
+ } catch (Exception e) {
+ log.error("error loading runtime library", e);
+ }
l.add(rtl);
}
return l;
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 1c7234a..d314fc5 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -1000,6 +1000,9 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
// Initialize the RestManager
restManager = initRestManager();
+ // at this point we can load jars loaded from remote urls.
+ memClassLoader.loadRemoteJars();
+
// Finally tell anyone who wants to know
resourceLoader.inform(resourceLoader);
resourceLoader.inform(this); // last call before the latch is released.
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 7269358..eacd758 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -62,6 +62,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigOverlay;
+import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
@@ -525,6 +526,15 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
op.getMap(PluginInfo.INVARIANTS, null);
op.getMap(PluginInfo.APPENDS, null);
if (op.hasError()) return overlay;
+ if(info.clazz == PluginBag.RuntimeLib.class) {
+ try {
+ new PluginBag.RuntimeLib(req.getCore()).init(new PluginInfo(info.tag, op.getDataMap()));
+ } catch (Exception e) {
+ op.addError(e.getMessage());
+ log.error("can't load this plugin ", e);
+ return overlay;
+ }
+ }
if (!verifyClass(op, clz, info.clazz)) return overlay;
if (pluginExists(info, overlay, name)) {
if (isCeate) {
diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java
index 7b4dc06..4a0f1ba 100644
--- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java
+++ b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.StringWriter;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.IOUtils;
@@ -31,11 +32,17 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.mockito.Mockito.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class BlobRepositoryMockingTest {
@@ -55,6 +62,8 @@ public class BlobRepositoryMockingTest {
ByteBuffer blobData = ByteBuffer.wrap(BLOBSTR.getBytes(UTF8));
boolean blobFetched = false;
String blobKey = "";
+ String url = null;
+ ByteBuffer filecontent = null;
@BeforeClass
public static void beforeClass() {
@@ -75,6 +84,14 @@ public class BlobRepositoryMockingTest {
}
@Override
+ ByteBuffer fetchFromUrl(String key, String url) {
+ if(!Objects.equals(url, BlobRepositoryMockingTest.this.url)) return null;
+ blobKey = key;
+ blobFetched = true;
+ return filecontent;
+ }
+
+ @Override
ConcurrentHashMap<String, BlobContent> createMap() {
return mapMock;
}
@@ -109,6 +126,31 @@ public class BlobRepositoryMockingTest {
@SuppressWarnings("unchecked")
@Test
+ public void testGetBlobIncrRefByUrl() throws Exception{
+ when(mockContainer.isZooKeeperAware()).thenReturn(true);
+ filecontent = TestDynamicLoading.getFileContent("runtimecode/runtimelibs_v2.jar.bin");
+ url = "http://localhost:8080/myjar/location.jar";
+ BlobRepository.BlobContentRef ref = repository.getBlobIncRef( "filefoo",null,url,
+ "bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417");
+ assertTrue("filefoo".equals(blobKey));
+ assertTrue(blobFetched);
+ assertNotNull(ref.blob);
+ assertEquals(filecontent, ref.blob.get());
+ verify(mockContainer).isZooKeeperAware();
+ try {
+ repository.getBlobIncRef( "filefoo",null,url,
+ "WRONG-SHA512-KEY");
+ fail("expected exception");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains(" expected sha512 hash : WRONG-SHA512-KEY , actual :"));
+ }
+
+ url = null;
+ filecontent = null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
public void testCachedAlready() {
when(mockContainer.isZooKeeperAware()).thenReturn(true);
when(mapMock.get("foo!")).thenReturn(new BlobRepository.BlobContent<BlobRepository>("foo!", blobData));
diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java
new file mode 100644
index 0000000..d202bb2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.handler.ReplicationHandler;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.RestTestHarness;
+import org.junit.BeforeClass;
+
+import static java.util.Arrays.asList;
+import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
+
+public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase {
+
+ @BeforeClass
+ public static void enableRuntimeLib() throws Exception {
+ System.setProperty("enable.runtime.lib", "true");
+ }
+
+ public static class JarHandler extends RequestHandlerBase {
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ rsp.add(ReplicationHandler.FILE_STREAM, (SolrCore.RawWriter) os -> {
+
+ ByteBuffer buf = TestDynamicLoading.getFileContent("runtimecode/runtimelibs.jar.bin");
+ if (buf == null) {
+ //should never happen unless a user wrote this document directly
+ throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Invalid document . No field called blob");
+ } else {
+ os.write(buf.array(), 0, buf.limit());
+ }
+ });
+
+ }
+
+ @Override
+ public String getDescription() {
+ return "serves jar files";
+ }
+ }
+
+
+ public void testDynamicLoadingUrl() throws Exception {
+ setupRestTestHarnesses();
+ String payload = "{\n" +
+ "'create-requesthandler' : { 'name' : '/jarhandler', 'class': " + JarHandler.class.getName() +
+ ", registerPath: '/solr,/v2' }\n" +
+ "}";
+
+ RestTestHarness client = randomRestTestHarness();
+ TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
+ TestSolrConfigHandler.testForResponseElement(client,
+ null,
+ "/config/overlay",
+ null,
+ Arrays.asList("overlay", "requestHandler", "/jarhandler", "class"),
+ JarHandler.class.getName(), 10);
+ payload = "{\n" +
+ "'add-runtimelib' : { 'name' : 'urljar', url : '" + client.getBaseURL() + "/jarhandler?wt=filestream'" +
+ " 'sha512':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
+ "}";
+ client = randomRestTestHarness();
+ TestSolrConfigHandler.runConfigCommandExpectFailure(client, "/config", payload, "Invalid jar");
+
+
+// String url = client
+ payload = "{\n" +
+ "'add-runtimelib' : { 'name' : 'urljar', url : '" + client.getBaseURL() + "/jarhandler?wt=filestream'" +
+ " 'sha512':'d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
+ "}";
+ client = randomRestTestHarness();
+ TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
+ TestSolrConfigHandler.testForResponseElement(client,
+ null,
+ "/config/overlay",
+ null,
+ Arrays.asList("overlay", "runtimeLib", "urljar", "sha512"),
+ "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", 10);
+
+ payload = "{\n" +
+ "'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}" +
+ "}";
+ client = randomRestTestHarness();
+ TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
+
+ TestSolrConfigHandler.testForResponseElement(client,
+ null,
+ "/config/overlay",
+ null,
+ Arrays.asList("overlay", "requestHandler", "/runtime", "class"),
+ "org.apache.solr.core.RuntimeLibReqHandler", 10);
+
+ Map result = TestSolrConfigHandler.testForResponseElement(client,
+ null,
+ "/runtime",
+ null,
+ Arrays.asList("class"),
+ "org.apache.solr.core.RuntimeLibReqHandler", 10);
+ compareValues(result, MemClassLoader.class.getName(), asList("loader"));
+
+
+ }
+}
+
+ /*
+ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true
+ openssl dgst -sha512 runtimelibs.jar
+ bin/solr start -e cloud -a "-Denable.runtime.lib=true" -noprompt
+
+ curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
+ "add-runtimelib": { "name" : "testjar",
+ "url":"http://localhost:8000/runtimelibs.jar" ,
+ "sha512" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"}
+ }'
+
+
+ curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
+ "delete-runtimelib": "testjar"}'
+
+ curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
+ "add-runtimelib": { "name" : "testjar",
+ "url":"https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin" ,
+ "sha512" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"}
+ }'
+
+curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
+ "create-requesthandler": { "name" : "/test",
+ 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}
+ }'
+*/
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
index 6e95fb5..17494e0 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
@@ -171,13 +171,14 @@ public class TestSolrConfigHandler extends RestTestBase {
reqhandlertests(restTestHarness, null, null);
}
- public static void runConfigCommand(RestTestHarness harness, String uri, String payload) throws IOException {
+ public static Map runConfigCommand(RestTestHarness harness, String uri, String payload) throws IOException {
String json = SolrTestCaseJ4.json(payload);
log.info("going to send config command. path {} , payload: {}", uri, payload);
String response = harness.post(uri, json);
Map map = (Map) Utils.fromJSONString(response);
assertNull(response, map.get("errorMessages"));
assertNull(response, map.get("errors")); // Will this ever be returned?
+ return map;
}
public static void runConfigCommandExpectFailure(RestTestHarness harness, String uri, String payload, String expectedErrorMessage) throws Exception {
diff --git a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc
index 234ce9b..096414d 100644
--- a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc
+++ b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc
@@ -30,7 +30,9 @@ Before enabling this feature, users should carefully consider the issues discuss
== Uploading Jar Files
-The first step is to use the <<blob-store-api.adoc#blob-store-api,Blob Store API>> to upload your jar files. This will to put your jars in the `.system` collection and distribute them across your SolrCloud nodes. These jars are added to a separate classloader and only accessible to components that are configured with the property `runtimeLib=true`. These components are loaded lazily because the `.system` collection may not be loaded when a particular core is loaded.
+Use your own service to host the jars or you can use Solr itself to host the jars.
+
+Use the <<blob-store-api.adoc#blob-store-api,Blob Store API>> to upload your jar files to Solr. This will to put your jars in the `.system` collection and distribute them across your SolrCloud nodes. These jars are added to a separate classloader and only accessible to components that are configured with the property `runtimeLib=true`. These components are loaded lazily because the `.system` collection may not be loaded when a particular core is loaded.
== Config API Commands to use Jars as Runtime Libraries
@@ -116,6 +118,79 @@ curl http://localhost:8983/api/collections/techproducts/config -H 'Content-type:
====
--
+== Using external service to host your jars
+
+Step 1: download a jar from github
+
+[source,bash]
+----
+ curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true
+
+----
+Step 2 get the `sha512` hash of the jar
+
+[source,bash]
+----
+ openssl dgst -sha512 runtimelibs.jar
+----
+
+Step 3 : start solr with runtime lib enabled
+
+[source,bash]
+----
+ bin/solr start -e cloud -a "-Denable.runtime.lib=true" -noprompt
+----
+
+Step 4: run a local server. skip this step if you have another place to host your jars
+
+[source,bash]
+----
+ python -m SimpleHTTPServer 8000 &
+----
+
+Step 5: add the jar to your collection `gettingstarted`
+[source,bash]
+----
+ #add the jar to your collection 'gettingstarted'
+ curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
+ "add-runtimelib": { "name" : "testjar",
+ "url":"http://localhost:8000/runtimelibs.jar" ,
+ "sha512" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"}
+ }'
+----
+
+Step 6 : create a new request handler '/test' for the collection 'gettingstarted' from the jar we just added
+[source,bash]
+----
+curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
+ "create-requesthandler": { "name" : "/test",
+ 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}
+ }'
+----
+
+Step 7: test your request handler
+[source,bash]
+----
+curl http://localhost:8983/solr/gettingstarted/test
+----
+
+output:
+[source,json]
+----
+{
+ "responseHeader":{
+ "status":0,
+ "QTime":0},
+ "params":{},
+ "context":{
+ "webapp":"/solr",
+ "path":"/test",
+ "httpMethod":"GET"},
+ "class":"org.apache.solr.core.RuntimeLibReqHandler",
+ "loader":"org.apache.solr.core.MemClassLoader"}
+----
+
+
== Securing Runtime Libraries
A drawback of this feature is that it could be used to load malicious executable code into the system. However, it is possible to restrict the system to load only trusted jars using http://en.wikipedia.org/wiki/Public_key_infrastructure[PKI] to verify that the executables loaded into the system are trustworthy.