You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/07/02 10:11:12 UTC
[lucene-solr] branch jira/SOLR-13565 updated: SOLR-13565: initial
commit
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/SOLR-13565
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-13565 by this push:
new e7ac801 SOLR-13565: initial commit
e7ac801 is described below
commit e7ac8012becbcc7c9160b615ebf67b8fecf93077
Author: noble <no...@apache.org>
AuthorDate: Tue Jul 2 20:10:49 2019 +1000
SOLR-13565: initial commit
---
.../java/org/apache/solr/core/BlobRepository.java | 4 +-
.../java/org/apache/solr/core/CoreContainer.java | 236 +++++++++------
.../src/java/org/apache/solr/core/LibListener.java | 98 ++++++
.../java/org/apache/solr/core/MemClassLoader.java | 16 +-
.../src/java/org/apache/solr/core/PluginBag.java | 168 ----------
.../src/java/org/apache/solr/core/RuntimeLib.java | 204 +++++++++++++
.../src/java/org/apache/solr/core/SolrConfig.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 2 +-
.../org/apache/solr/handler/SolrConfigHandler.java | 8 +-
.../solr/handler/admin/CollectionHandlerApi.java | 180 ++++++++---
.../apache/solr/cloud/TestClusterProperties.java | 136 ++++++++-
.../org/apache/solr/core/TestDynamicLoading.java | 2 +-
.../client/solrj/request/CollectionApiMapping.java | 17 ++
.../solr/common/cloud/ClusterProperties.java | 6 +
.../apache/solr/common/cloud/ZkStateReader.java | 5 +-
.../java/org/apache/solr/common/util/Utils.java | 337 +++++++++++----------
.../src/resources/apispec/cluster.Commands.json | 37 +++
.../cluster.Commands.runtimelib.properties.json | 23 ++
18 files changed, 1006 insertions(+), 475 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
index 24bb88e..b8ced48 100644
--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
@@ -60,7 +60,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
* The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the Jar in a single node.
*/
public class BlobRepository {
- private static final long MAX_JAR_SIZE = Long.parseLong(System.getProperty("runtme.lib.size", String.valueOf(5 * 1024 * 1024)));
+ private static final long MAX_JAR_SIZE = Long.parseLong(System.getProperty("runtime.lib.size", String.valueOf(5 * 1024 * 1024)));
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
static final Random RANDOM;
static final Pattern BLOB_KEY_PATTERN_CHECKER = Pattern.compile(".*/\\d+");
@@ -214,7 +214,7 @@ public class BlobRepository {
entity = response.getEntity();
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
- throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such blob or version available: " + key);
+ throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such resource available: " + key + ", url : "+ url);
}
try (InputStream is = entity.getContent()) {
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 3552343..f9a7281 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -33,6 +33,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -72,6 +73,7 @@ import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.backup.repository.BackupRepository;
@@ -130,7 +132,9 @@ import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
+import static org.apache.solr.schema.FieldType.CLASS_NAME;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
/**
@@ -218,6 +222,8 @@ public class CoreContainer {
protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
+ private final LibListener clusterPropertiesListener = new LibListener(this);
+
// Bits for the state variable.
public final static long LOAD_COMPLETE = 0x1L;
@@ -337,40 +343,7 @@ public class CoreContainer {
cfg.getReplayUpdatesThreads(),
new DefaultSolrThreadFactory("replayUpdatesExecutor")));
}
-
- private synchronized void initializeAuthorizationPlugin(Map<String, Object> authorizationConf) {
- authorizationConf = Utils.getDeepCopy(authorizationConf, 4);
- int newVersion = readVersion(authorizationConf);
- //Initialize the Authorization module
- SecurityPluginHolder<AuthorizationPlugin> old = authorizationPlugin;
- SecurityPluginHolder<AuthorizationPlugin> authorizationPlugin = null;
- if (authorizationConf != null) {
- String klas = (String) authorizationConf.get("class");
- if (klas == null) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "class is required for authorization plugin");
- }
- if (old != null && old.getZnodeVersion() == newVersion && newVersion > 0) {
- log.debug("Authorization config not modified");
- return;
- }
- log.info("Initializing authorization plugin: " + klas);
- authorizationPlugin = new SecurityPluginHolder<>(newVersion,
- getResourceLoader().newInstance(klas, AuthorizationPlugin.class));
-
- // Read and pass the authorization context to the plugin
- authorizationPlugin.plugin.init(authorizationConf);
- } else {
- log.debug("Security conf doesn't exist. Skipping setup for authorization module.");
- }
- this.authorizationPlugin = authorizationPlugin;
- if (old != null) {
- try {
- old.plugin.close();
- } catch (Exception e) {
- log.error("Exception while attempting to close old authorization plugin", e);
- }
- }
- }
+ private Map<String, PluginInfo> runtimeHandlers = new HashMap<>();
private void initializeAuditloggerPlugin(Map<String, Object> auditConf) {
auditConf = Utils.getDeepCopy(auditConf, 4);
@@ -405,6 +378,39 @@ public class CoreContainer {
}
}
+ private synchronized void initializeAuthorizationPlugin(Map<String, Object> authorizationConf) {
+ authorizationConf = Utils.getDeepCopy(authorizationConf, 4);
+ int newVersion = readVersion(authorizationConf);
+ //Initialize the Authorization module
+ SecurityPluginHolder<AuthorizationPlugin> old = authorizationPlugin;
+ SecurityPluginHolder<AuthorizationPlugin> authorizationPlugin = null;
+ if (authorizationConf != null) {
+ String klas = (String) authorizationConf.get("class");
+ if (klas == null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "class is required for authorization plugin");
+ }
+ if (old != null && old.getZnodeVersion() == newVersion && newVersion > 0) {
+ log.debug("Authorization config not modified");
+ return;
+ }
+ log.info("Initializing authorization plugin: " + klas);
+ authorizationPlugin = new SecurityPluginHolder<>(newVersion,
+ createInstance(klas, AuthorizationPlugin.class));
+
+ // Read and pass the authorization context to the plugin
+ authorizationPlugin.plugin.init(authorizationConf);
+ } else {
+ log.debug("Security conf doesn't exist. Skipping setup for authorization module.");
+ }
+ this.authorizationPlugin = authorizationPlugin;
+ if (old != null) {
+ try {
+ old.plugin.close();
+ } catch (Exception e) {
+ log.error("Exception while attempting to close old authorization plugin", e);
+ }
+ }
+ }
private synchronized void initializeAuthenticationPlugin(Map<String, Object> authenticationConfig) {
authenticationConfig = Utils.getDeepCopy(authenticationConfig, 4);
@@ -439,11 +445,7 @@ public class CoreContainer {
if (pluginClassName != null) {
log.info("Initializing authentication plugin: " + pluginClassName);
authenticationPlugin = new SecurityPluginHolder<>(newVersion,
- getResourceLoader().newInstance(pluginClassName,
- AuthenticationPlugin.class,
- null,
- new Class[]{CoreContainer.class},
- new Object[]{this}));
+ createInstance(pluginClassName, AuthenticationPlugin.class));
}
if (authenticationPlugin != null) {
authenticationPlugin.plugin.init(authenticationConfig);
@@ -458,6 +460,7 @@ public class CoreContainer {
}
+
private void setupHttpClientForAuthPlugin(Object authcPlugin) {
if (authcPlugin instanceof HttpClientBuilderPlugin) {
// Setup HttpClient for internode communication
@@ -577,6 +580,39 @@ public class CoreContainer {
// Initialization / Cleanup
//-------------------------------------------------------------------
+ public <T> T createInstance(String cName, Class<T> expectedType) {
+ try {
+ return getResourceLoader().newInstance(cName, expectedType, null, new Class[]{CoreContainer.class}, new Object[]{this});
+ } catch (SolrException e) {
+ // try with memclassloader
+ MemClassLoader memClassLoader = clusterPropertiesListener.memClassLoader;
+ if (memClassLoader != null) {
+ try {
+ Class<? extends T> klas = memClassLoader.findClass(cName, expectedType);
+ try {
+ return klas.getConstructor(CoreContainer.class).newInstance(this);
+
+ } catch (NoSuchMethodException ex) {
+ return klas.getConstructor(null).newInstance(null);
+ }
+
+ } catch (Exception ex) {
+ if (!memClassLoader.getErrors().isEmpty()) {
+ //may be this class was supposed to be loaded from a runtime lib
+ throw new SolrException(ErrorCode.SERVER_ERROR, "error loading class from runtime library " + StrUtils.join(memClassLoader.getErrors(), ','), ex);
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "error loading class from runtime library", ex);
+
+ }
+ }
+ } else {
+ throw e;
+ }
+
+ }
+
+ }
+
/**
* Load the cores defined for this CoreContainer
*/
@@ -622,6 +658,7 @@ public class CoreContainer {
zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig());
if (isZooKeeperAware()) {
+// getZkController().getZkStateReader().registerClusterPropertiesListener(clusterPropertiesListener);
pkiAuthenticationPlugin = new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName(),
(PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH));
TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader());
@@ -635,10 +672,8 @@ public class CoreContainer {
createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);
- collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
- infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
- coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
- configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
+
+ initConfigurableHandlers(null);
// metricsHistoryHandler uses metricsHandler, so create it first
metricsHandler = new MetricsHandler(this);
@@ -809,49 +844,15 @@ public class CoreContainer {
status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
}
- // MetricsHistoryHandler supports both cloud and standalone configs
- private void createMetricsHistoryHandler() {
- PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
- Map<String, Object> initArgs;
- if (plugin != null && plugin.initArgs != null) {
- initArgs = plugin.initArgs.asMap(5);
- initArgs.put(MetricsHistoryHandler.ENABLE_PROP, plugin.isEnabled());
- } else {
- initArgs = new HashMap<>();
- }
- String name;
- SolrCloudManager cloudManager;
- SolrClient client;
- if (isZooKeeperAware()) {
- name = getZkController().getNodeName();
- cloudManager = getZkController().getSolrCloudManager();
- client = new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()), Optional.empty())
- .withSocketTimeout(30000).withConnectionTimeout(15000)
- .withHttpClient(updateShardHandler.getDefaultHttpClient()).build();
- } else {
- name = getNodeConfig().getNodeName();
- if (name == null || name.isEmpty()) {
- name = "localhost";
- }
- cloudManager = null;
- client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL) {
- @Override
- public void close() throws IOException {
- // do nothing - we close the container ourselves
- }
- };
- // enable local metrics unless specifically set otherwise
- if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_NODES_PROP)) {
- initArgs.put(MetricsHistoryHandler.ENABLE_NODES_PROP, true);
- }
- if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_REPLICAS_PROP)) {
- initArgs.put(MetricsHistoryHandler.ENABLE_REPLICAS_PROP, true);
- }
- }
- metricsHistoryHandler = new MetricsHistoryHandler(name, metricsHandler,
- client, cloudManager, initArgs);
- containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
- metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_HISTORY_PATH);
+ private void initConfigurableHandlers(Set<String> handlers) {
+ if(handlers == null || handlers.contains(COLLECTIONS_HANDLER_PATH))
+ collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
+ if(handlers == null || handlers.contains(INFO_HANDLER_PATH))
+ infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
+ if(handlers == null || handlers.contains(CORES_HANDLER_PATH))
+ coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
+ if(handlers == null || handlers.contains(CORES_HANDLER_PATH))
+ configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
}
public void securityNodeChanged() {
@@ -1770,10 +1771,59 @@ public class CoreContainer {
// ---------------- CoreContainer request handlers --------------
+ // MetricsHistoryHandler supports both cloud and standalone configs
+ private void createMetricsHistoryHandler() {
+ PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
+ Map<String, Object> initArgs;
+ if (plugin != null && plugin.initArgs != null) {
+ initArgs = plugin.initArgs.asMap(5);
+ initArgs.put(MetricsHistoryHandler.ENABLE_PROP, plugin.isEnabled());
+ } else {
+ initArgs = new HashMap<>();
+ }
+ String name;
+ SolrCloudManager cloudManager;
+ SolrClient client;
+ if (isZooKeeperAware()) {
+ name = getZkController().getNodeName();
+ cloudManager = getZkController().getSolrCloudManager();
+ client = new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()), Optional.empty())
+ .withSocketTimeout(30000).withConnectionTimeout(15000)
+ .withHttpClient(updateShardHandler.getDefaultHttpClient()).build();
+ } else {
+ name = getNodeConfig().getNodeName();
+ if (name == null || name.isEmpty()) {
+ name = "localhost";
+ }
+ cloudManager = null;
+ client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL) {
+ @Override
+ public void close() throws IOException {
+ // do nothing - we close the container ourselves
+ }
+ };
+ // enable local metrics unless specifically set otherwise
+ if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_NODES_PROP)) {
+ initArgs.put(MetricsHistoryHandler.ENABLE_NODES_PROP, true);
+ }
+ if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_REPLICAS_PROP)) {
+ initArgs.put(MetricsHistoryHandler.ENABLE_REPLICAS_PROP, true);
+ }
+ }
+ MetricsHistoryHandler metricsHistoryHandler = new MetricsHistoryHandler(name, metricsHandler,
+ client, cloudManager, initArgs);
+ containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
+ metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_HISTORY_PATH);
+ }
+
protected <T> T createHandler(String path, String handlerClass, Class<T> clazz) {
- T handler = loader.newInstance(handlerClass, clazz, null, new Class[]{CoreContainer.class}, new Object[]{this});
+ T handler = createInstance(handlerClass, clazz);
if (handler instanceof SolrRequestHandler) {
containerHandlers.put(path, (SolrRequestHandler) handler);
+ if (handler.getClass().getClassLoader() instanceof MemClassLoader) {
+ runtimeHandlers.put(path, new PluginInfo(SolrRequestHandler.TYPE,
+ Utils.makeMap(NAME,path, CLASS_NAME, handlerClass)));
+ }
}
if (handler instanceof SolrMetricProducer) {
((SolrMetricProducer) handler).initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, path);
@@ -1995,6 +2045,18 @@ public class CoreContainer {
public void runAsync(Runnable r) {
coreContainerAsyncTaskExecutor.submit(r);
}
+
+ synchronized void reloadAllComponents() {
+ if(runtimeHandlers.isEmpty()) return;
+ initConfigurableHandlers(runtimeHandlers.keySet());
+ boolean reloadSecurity = false;
+ if(authenticationPlugin != null && auditloggerPlugin.getClass().getClassLoader() instanceof MemClassLoader) reloadSecurity = true;
+ if(authorizationPlugin != null && authorizationPlugin.getClass().getClassLoader() instanceof MemClassLoader) reloadSecurity = true;
+ if(reloadSecurity){
+ reloadSecurityProperties();
+ }
+ }
+
}
class CloserThread extends Thread {
diff --git a/solr/core/src/java/org/apache/solr/core/LibListener.java b/solr/core/src/java/org/apache/solr/core/LibListener.java
new file mode 100644
index 0000000..0cfc4f7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/LibListener.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.cloud.ClusterPropertiesListener;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class LibListener implements ClusterPropertiesListener {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CoreContainer coreContainer;
+
+ Map<String, RuntimeLib> runtimeLibs = new HashMap<>();
+ MemClassLoader memClassLoader;
+
+ LibListener(CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+ }
+
+
+ @Override
+ public boolean onChange(Map<String, Object> properties) {
+ Map m = (Map) properties.getOrDefault("runtimeLib", Collections.emptyMap());
+ boolean needsReload[] = new boolean[1];
+ if (m.size() == runtimeLibs.size()) {
+ m.forEach((k, v) -> {
+ if(!runtimeLibs.containsKey(k)) needsReload[0] = true;
+
+ });
+ } else {
+ needsReload[0] = true;
+ }
+ if(needsReload[0]){
+ createNewClassLoader(m);
+ }
+
+ return false;
+ }
+ void createNewClassLoader(Map m) {
+ List<RuntimeLib> libs = new ArrayList<>(m.size());
+ boolean[] loadedAll = new boolean[1];
+ loadedAll[0] = true;
+ Map<String , RuntimeLib> libMap = new LinkedHashMap<>();
+ m.forEach((k, v) -> {
+ if (v instanceof Map) {
+ Map map = new HashMap((Map) v);
+ map.put(CoreAdminParams.NAME, String.valueOf(k));
+ RuntimeLib lib = new RuntimeLib(coreContainer);
+ try {
+ lib.init(new PluginInfo(null,map));
+ if(lib.getUrl() == null){
+ log.error("Unable to initialize runtimeLib : "+ Utils.toJSONString(v));
+ loadedAll[0] = false;
+ }
+ lib.loadJar();
+ libMap.put(lib.getName(), lib);
+ } catch (Exception e) {
+ log.error("error loading a runtimeLib "+Utils.toJSONString(v), e );
+ loadedAll[0] = false;
+
+ }
+ }
+ });
+
+ if(loadedAll[0] ){
+ this.runtimeLibs = libMap;
+ this.memClassLoader = new MemClassLoader( libs, coreContainer.getResourceLoader());
+ coreContainer.reloadAllComponents();
+ }
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
index cf6bb4d..7bba195 100644
--- a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
@@ -26,6 +26,7 @@ import java.security.CodeSource;
import java.security.ProtectionDomain;
import java.security.cert.Certificate;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -43,12 +44,12 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private boolean allJarsLoaded = false;
private final SolrResourceLoader parentLoader;
- private List<PluginBag.RuntimeLib> libs = new ArrayList<>();
+ private List<RuntimeLib> libs = new ArrayList<>();
private Map<String, Class> classCache = new HashMap<>();
private List<String> errors = new ArrayList<>();
- public MemClassLoader(List<PluginBag.RuntimeLib> libs, SolrResourceLoader resourceLoader) {
+ public MemClassLoader(List<RuntimeLib> libs, SolrResourceLoader resourceLoader) {
this.parentLoader = resourceLoader;
this.libs = libs;
}
@@ -56,7 +57,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
synchronized void loadRemoteJars() {
if (allJarsLoaded) return;
int count = 0;
- for (PluginBag.RuntimeLib lib : libs) {
+ for (RuntimeLib lib : libs) {
if (lib.getUrl() != null) {
try {
lib.loadJar();
@@ -70,10 +71,13 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
if (count == libs.size()) allJarsLoaded = true;
}
+ public Collection<String> getErrors(){
+ return errors;
+ }
public synchronized void loadJars() {
if (allJarsLoaded) return;
- for (PluginBag.RuntimeLib lib : libs) {
+ for (RuntimeLib lib : libs) {
try {
lib.loadJar();
lib.verify();
@@ -133,7 +137,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
String path = name.replace('.', '/').concat(".class");
ByteBuffer buf = null;
- for (PluginBag.RuntimeLib lib : libs) {
+ for (RuntimeLib lib : libs) {
try {
buf = lib.getFileContent(path);
if (buf != null) {
@@ -150,7 +154,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable, Resour
@Override
public void close() throws Exception {
- for (PluginBag.RuntimeLib lib : libs) {
+ for (RuntimeLib lib : libs) {
try {
lib.close();
} catch (Exception e) {
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index 977a7e6..6f3525a 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -16,11 +16,8 @@
*/
package org.apache.solr.core;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -30,15 +27,12 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
-import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.RequestHandlerBase;
@@ -46,8 +40,6 @@ import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
-import org.apache.solr.util.CryptoKeys;
-import org.apache.solr.util.SimplePostTool;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -56,7 +48,6 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonMap;
import static org.apache.solr.api.ApiBag.HANDLER_NAME;
-import static org.apache.solr.common.params.CommonParams.NAME;
/**
* This manages the lifecycle of a set of plugin of the same type .
@@ -460,165 +451,6 @@ public class PluginBag<T> implements AutoCloseable {
}
- /**
- * This represents a Runtime Jar. A jar requires two details , name and version
- */
- public static class RuntimeLib implements PluginInfoInitialized, AutoCloseable {
- private String name, version, sig, sha512, url;
- private BlobRepository.BlobContentRef<ByteBuffer> jarContent;
- private final CoreContainer coreContainer;
- private boolean verified = false;
-
- @Override
- public void init(PluginInfo info) {
- name = info.attributes.get(NAME);
- url = info.attributes.get("url");
- sig = info.attributes.get("sig");
- if(url == null) {
- Object v = info.attributes.get("version");
- if (name == null || v == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "runtimeLib must have name and version");
- }
- version = String.valueOf(v);
- } else {
- sha512 = info.attributes.get("sha512");
- if(sha512 == null){
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute");
- }
- ByteBuffer buf = null;
- buf = coreContainer.getBlobRepository().fetchFromUrl(name, url);
-
- String digest = BlobRepository.sha512Digest(buf);
- if(!sha512.equals(digest)) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest) );
- }
- log.info("dynamic library verified {}, sha512: {}", url, sha512);
-
- }
-
- }
-
- public RuntimeLib(SolrCore core) {
- coreContainer = core.getCoreContainer();
- }
-
- public String getUrl(){
- return url;
- }
-
- void loadJar() {
- if (jarContent != null) return;
- synchronized (this) {
- if (jarContent != null) return;
-
- jarContent = url == null?
- coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version):
- coreContainer.getBlobRepository().getBlobIncRef(name, null,url,sha512);
-
- }
- }
-
- public static boolean isEnabled() {
- return "true".equals(System.getProperty("enable.runtime.lib"));
- }
-
- public String getName() {
- return name;
- }
-
- public String getVersion() {
- return version;
- }
-
- public String getSig() {
- return sig;
-
- }
-
- public ByteBuffer getFileContent(String entryName) throws IOException {
- if (jarContent == null)
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name );
- return getFileContent(jarContent.blob, entryName);
-
- }
-
- public ByteBuffer getFileContent(BlobRepository.BlobContent<ByteBuffer> blobContent, String entryName) throws IOException {
- ByteBuffer buff = blobContent.get();
- ByteArrayInputStream zipContents = new ByteArrayInputStream(buff.array(), buff.arrayOffset(), buff.limit());
- ZipInputStream zis = new ZipInputStream(zipContents);
- try {
- ZipEntry entry;
- while ((entry = zis.getNextEntry()) != null) {
- if (entryName == null || entryName.equals(entry.getName())) {
- SimplePostTool.BAOS out = new SimplePostTool.BAOS();
- byte[] buffer = new byte[2048];
- int size;
- while ((size = zis.read(buffer, 0, buffer.length)) != -1) {
- out.write(buffer, 0, size);
- }
- out.close();
- return out.getByteBuffer();
- }
- }
- } finally {
- zis.closeEntry();
- }
- return null;
- }
-
-
- @Override
- public void close() throws Exception {
- if (jarContent != null) coreContainer.getBlobRepository().decrementBlobRefCount(jarContent);
- }
-
- public static List<RuntimeLib> getLibObjects(SolrCore core, List<PluginInfo> libs) {
- List<RuntimeLib> l = new ArrayList<>(libs.size());
- for (PluginInfo lib : libs) {
- RuntimeLib rtl = new RuntimeLib(core);
- try {
- rtl.init(lib);
- } catch (Exception e) {
- log.error("error loading runtime library", e);
- }
- l.add(rtl);
- }
- return l;
- }
-
- public void verify() throws Exception {
- if (verified) return;
- if (jarContent == null) {
- log.error("Calling verify before loading the jar");
- return;
- }
-
- if (!coreContainer.isZooKeeperAware())
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Signing jar is possible only in cloud");
- Map<String, byte[]> keys = CloudUtil.getTrustedKeys(coreContainer.getZkController().getZkClient(), "exe");
- if (keys.isEmpty()) {
- if (sig == null) {
- verified = true;
- return;
- } else {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No public keys are available in ZK to verify signature for runtime lib " + name);
- }
- } else if (sig == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString("runtimelib {0} should be signed with one of the keys in ZK /keys/exe ", name));
- }
-
- try {
- String matchedKey = new CryptoKeys(keys).verify(sig, jarContent.blob.get());
- if (matchedKey == null)
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No key matched signature for jar : " + name + " version: " + version);
- log.info("Jar {} signed with {} successfully verified", name, matchedKey);
- } catch (Exception e) {
- if (e instanceof SolrException) throw e;
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error verifying key ", e);
- }
- }
- }
-
public Api v2lookup(String path, String method, Map<String, String> parts) {
if (apiBag == null) {
diff --git a/solr/core/src/java/org/apache/solr/core/RuntimeLib.java b/solr/core/src/java/org/apache/solr/core/RuntimeLib.java
new file mode 100644
index 0000000..b112c53
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/RuntimeLib.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.solr.cloud.CloudUtil;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.params.CommonParams.NAME;
+
+/**
+ * This represents a Runtime Jar. A jar requires two details , name and version
+ */
+public class RuntimeLib implements PluginInfoInitialized, AutoCloseable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final CoreContainer coreContainer;
+ private String name, version, sig, sha512, url;
+ private BlobRepository.BlobContentRef<ByteBuffer> jarContent;
+ private boolean verified = false;
+
+ public RuntimeLib(CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+ }
+
+ public static boolean isEnabled() {
+ return "true".equals(System.getProperty("enable.runtime.lib"));
+ }
+
+ public static List<RuntimeLib> getLibObjects(SolrCore core, List<PluginInfo> libs) {
+ List<RuntimeLib> l = new ArrayList<>(libs.size());
+ for (PluginInfo lib : libs) {
+ RuntimeLib rtl = new RuntimeLib(core.getCoreContainer());
+ try {
+ rtl.init(lib);
+ } catch (Exception e) {
+ log.error("error loading runtime library", e);
+ }
+ l.add(rtl);
+ }
+ return l;
+ }
+
+ @Override
+ public void init(PluginInfo info) {
+ name = info.attributes.get(NAME);
+ url = info.attributes.get("url");
+ sig = info.attributes.get("sig");
+ if(url == null) {
+ Object v = info.attributes.get("version");
+ if (name == null || v == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "runtimeLib must have name and version");
+ }
+ version = String.valueOf(v);
+ } else {
+ sha512 = info.attributes.get("sha512");
+ if(sha512 == null){
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute");
+ }
+ ByteBuffer buf = null;
+ buf = coreContainer.getBlobRepository().fetchFromUrl(name, url);
+
+ String digest = BlobRepository.sha512Digest(buf);
+ if(!sha512.equals(digest)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest) );
+ }
+ verifyJarSignature();
+
+ log.debug("dynamic library verified {}, sha512: {}", url, sha512);
+
+ }
+
+ }
+
+ public String getUrl(){
+ return url;
+ }
+
+ void loadJar() {
+ if (jarContent != null) return;
+ synchronized (this) {
+ if (jarContent != null) return;
+
+ jarContent = url == null?
+ coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version):
+ coreContainer.getBlobRepository().getBlobIncRef(name, null,url,sha512);
+
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public String getSig() {
+ return sig;
+
+ }
+
+ public ByteBuffer getFileContent(String entryName) throws IOException {
+ if (jarContent == null)
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "jar not available: " + name );
+ return getFileContent(jarContent.blob, entryName);
+
+ }
+
+ public ByteBuffer getFileContent(BlobRepository.BlobContent<ByteBuffer> blobContent, String entryName) throws IOException {
+ ByteBuffer buff = blobContent.get();
+ ByteArrayInputStream zipContents = new ByteArrayInputStream(buff.array(), buff.arrayOffset(), buff.limit());
+ ZipInputStream zis = new ZipInputStream(zipContents);
+ try {
+ ZipEntry entry;
+ while ((entry = zis.getNextEntry()) != null) {
+ if (entryName == null || entryName.equals(entry.getName())) {
+ SimplePostTool.BAOS out = new SimplePostTool.BAOS();
+ byte[] buffer = new byte[2048];
+ int size;
+ while ((size = zis.read(buffer, 0, buffer.length)) != -1) {
+ out.write(buffer, 0, size);
+ }
+ out.close();
+ return out.getByteBuffer();
+ }
+ }
+ } finally {
+ zis.closeEntry();
+ }
+ return null;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (jarContent != null) coreContainer.getBlobRepository().decrementBlobRefCount(jarContent);
+ }
+
+ public void verify() throws Exception {
+ if (verified) return;
+ if (jarContent == null) {
+ log.error("Calling verify before loading the jar");
+ return;
+ }
+
+ if (!coreContainer.isZooKeeperAware())
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Signing jar is possible only in cloud");
+ verifyJarSignature();
+ }
+
+ void verifyJarSignature() {
+ Map<String, byte[]> keys = CloudUtil.getTrustedKeys(coreContainer.getZkController().getZkClient(), "exe");
+ if (keys.isEmpty()) {
+ if (sig == null) {
+ verified = true;
+ return;
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No public keys are available in ZK to verify signature for runtime lib " + name);
+ }
+ } else if (sig == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString("runtimelib {0} should be signed with one of the keys in ZK /keys/exe ", name));
+ }
+
+ try {
+ String matchedKey = new CryptoKeys(keys).verify(sig, jarContent.blob.get());
+ if (matchedKey == null)
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No key matched signature for jar : " + name + " version: " + version);
+ log.info("Jar {} signed with {} successfully verified", name, matchedKey);
+ } catch (Exception e) {
+ if (e instanceof SolrException) throw (SolrException)e;
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error verifying key ", e);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index 232543c..caa1b25 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -377,7 +377,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
// and even then -- only if there is a single SpellCheckComponent
// because of queryConverter.setIndexAnalyzer
.add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS))
- .add(new SolrPluginInfo(PluginBag.RuntimeLib.class, "runtimeLib", REQUIRE_NAME, MULTI_OK))
+ .add(new SolrPluginInfo(RuntimeLib.class, "runtimeLib", REQUIRE_NAME, MULTI_OK))
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index b730f3e..be29e7a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -966,7 +966,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
this.codec = initCodec(solrConfig, this.schema);
memClassLoader = new MemClassLoader(
- PluginBag.RuntimeLib.getLibObjects(this, solrConfig.getPluginInfos(PluginBag.RuntimeLib.class.getName())),
+ RuntimeLib.getLibObjects(this, solrConfig.getPluginInfos(RuntimeLib.class.getName())),
getResourceLoader());
initIndex(prev != null, reload);
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 11c6404..dd591e8 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -62,9 +62,9 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigOverlay;
-import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.RequestParams;
+import org.apache.solr.core.RuntimeLib;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
@@ -526,13 +526,13 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
op.getMap(PluginInfo.INVARIANTS, null);
op.getMap(PluginInfo.APPENDS, null);
if (op.hasError()) return overlay;
- if(info.clazz == PluginBag.RuntimeLib.class) {
- if(!PluginBag.RuntimeLib.isEnabled()){
+ if(info.clazz == RuntimeLib.class) {
+ if(!RuntimeLib.isEnabled()){
op.addError("Solr not started with -Denable.runtime.lib=true");
return overlay;
}
try {
- new PluginBag.RuntimeLib(req.getCore()).init(new PluginInfo(info.tag, op.getDataMap()));
+ new RuntimeLib(req.getCore().getCoreContainer()).init(new PluginInfo(info.tag, op.getDataMap()));
} catch (Exception e) {
op.addError(e.getMessage());
log.error("can't load this plugin ", e);
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
index d7d179a..0643cb0 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
@@ -18,39 +18,54 @@
package org.apache.solr.handler.admin;
import java.lang.invoke.MethodHandles;
-import java.util.Arrays;
import java.util.Collection;
import java.util.EnumMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionApiMapping;
import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta;
import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta;
import org.apache.solr.client.solrj.request.CollectionApiMapping.V2EndPoint;
-import org.apache.solr.common.Callable;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.RuntimeLib;
import org.apache.solr.handler.admin.CollectionsHandler.CollectionOperation;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static org.apache.solr.common.util.CommandOperation.captureErrors;
+
public class CollectionHandlerApi extends BaseHandlerApiSupport {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final CollectionsHandler handler;
static Collection<ApiCommand> apiCommands = createCollMapping();
+ public CollectionHandlerApi(CollectionsHandler handler) {
+ this.handler = handler;
+ }
+
private static Collection<ApiCommand> createCollMapping() {
- Map<Meta, ApiCommand> result = new EnumMap<>(Meta.class);
+ Map<Meta, ApiCommand> apiMapping = new EnumMap<>(Meta.class);
for (Meta meta : Meta.values()) {
for (CollectionOperation op : CollectionOperation.values()) {
if (op.action == meta.action) {
- result.put(meta, new ApiCommand() {
+ apiMapping.put(meta, new ApiCommand() {
@Override
public CommandMeta meta() {
return meta;
@@ -65,30 +80,106 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
}
}
//The following APIs have only V2 implementations
- addApi(result, Meta.GET_NODES, params -> params.rsp.add("nodes", ((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes()));
- addApi(result, Meta.SET_CLUSTER_PROPERTY_OBJ, params -> {
- List<CommandOperation> commands = params.req.getCommands(true);
- if (commands == null || commands.isEmpty()) throw new RuntimeException("Empty commands");
- ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
-
- try {
- clusterProperties.setClusterProperties(commands.get(0).getDataMap());
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e);
- }
- });
+ addApi(apiMapping, Meta.GET_NODES, CollectionHandlerApi::getNodes);
+ addApi(apiMapping, Meta.SET_CLUSTER_PROPERTY_OBJ, CollectionHandlerApi::setClusterObj);
+ addApi(apiMapping, Meta.ADD_RUNTIME_LIB, CollectionHandlerApi::addUpdateRuntimeLib);
+ addApi(apiMapping, Meta.UPDATE_RUNTIME_LIB, CollectionHandlerApi::addUpdateRuntimeLib);
+ addApi(apiMapping, Meta.ADD_REQ_HANDLER, CollectionHandlerApi::addRequestHandler);
+ addApi(apiMapping, Meta.DELETE_REQ_HANDLER, CollectionHandlerApi::deleteReqHandler);
for (Meta meta : Meta.values()) {
- if (result.get(meta) == null) {
+ if (apiMapping.get(meta) == null) {
log.error("ERROR_INIT. No corresponding API implementation for : " + meta.commandName);
}
}
- return result.values();
+ return apiMapping.values();
+ }
+
+ private static void getNodes(ApiInfo params) {
+ params.rsp.add("nodes", ((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes());
+ }
+
+ private static void deleteReqHandler(ApiInfo params) throws Exception {
+ String name = params.op.getStr("");
+ ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
+ Map<String, Object> map = clusterProperties.getClusterProperties();
+ if (Utils.getObjectByPath(map, false, asList(SolrRequestHandler.TYPE, name)) == null) {
+ params.op.addError("NO such requestHandler with name :");
+ return;
+ }
+ Map m = new LinkedHashMap();
+ Utils.setObjectByPath(m, asList(SolrRequestHandler.TYPE, name), null, true);
+ clusterProperties.setClusterProperties(m);
+ }
+
+ private static void addRequestHandler(ApiInfo params) throws Exception {
+ Map data = params.op.getDataMap();
+ String name = (String) data.get("name");
+ ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
+ Map<String, Object> map = clusterProperties.getClusterProperties();
+ if (Utils.getObjectByPath(map, false, asList(SolrRequestHandler.TYPE, name)) != null) {
+ params.op.addError("A requestHandler already exists with the said name");
+ return;
+ }
+ Map m = new LinkedHashMap();
+ Utils.setObjectByPath(m, asList(SolrRequestHandler.TYPE, name), data, true);
+ clusterProperties.setClusterProperties(m);
}
- private static void addApi(Map<Meta, ApiCommand> result, Meta metaInfo, Callable<ApiParams> fun) {
- result.put(metaInfo, new ApiCommand() {
+ private static CommandOperation getFirstCommand(ApiInfo params) {
+ List<CommandOperation> commands = params.req.getCommands(true);
+ if (commands == null || commands.size() != 1)
+ throw new SolrException(ErrorCode.BAD_REQUEST, "should have exactly one command");
+ return commands.get(0);
+ }
+
+ private static void addUpdateRuntimeLib(ApiInfo params) throws Exception {
+ CollectionHandlerApi handler = (CollectionHandlerApi) params.apiHandler;
+ RuntimeLib lib = new RuntimeLib(handler.handler.coreContainer);
+ CommandOperation op = params.op;
+ String name = op.getStr("name");
+ ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
+ Map<String, Object> props = clusterProperties.getClusterProperties();
+ List<String> pathToLib = asList("runtimeLib", name);
+ Map existing = (Map) Utils.getObjectByPath(props, false, pathToLib);
+ if( Meta.ADD_RUNTIME_LIB.commandName.equals(op.name)){
+ if(existing != null){
+ op.addError(StrUtils.formatString("The jar with a name ''{0}'' already exists",name));
+ return;
+ }
+ } else {
+ if(existing == null){
+ op.addError(StrUtils.formatString("The jar with a name ''{0}'' doesn not exist",name));
+ return;
+ }
+ }
+ try {
+ lib.init(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()));
+ } catch (SolrException e) {
+ log.error("Error loading runtimelib ", e);
+ op.addError(e.getMessage());
+ return;
+ }
+
+ Map delta = new LinkedHashMap();
+ Utils.setObjectByPath(delta, pathToLib, op.getDataMap(), true);
+ clusterProperties.setClusterProperties(delta);
+
+ }
+
+ private static void setClusterObj(ApiInfo params) {
+ ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
+ try {
+ clusterProperties.setClusterProperties(params.op.getDataMap());
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Error in API", e);
+ }
+ }
+
+ private static void addApi(Map<Meta, ApiCommand> mapping, Meta metaInfo, Command fun) {
+ mapping.put(metaInfo, new ApiCommand() {
+
@Override
public CommandMeta meta() {
return metaInfo;
@@ -96,35 +187,50 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
@Override
public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
- fun.call(new ApiParams(req, rsp, apiHandler));
+ CommandOperation op = null;
+ if(metaInfo.method == SolrRequest.METHOD.POST) {
+ List<CommandOperation> commands = req.getCommands(true);
+ if (commands == null || commands.size() != 1)
+ throw new SolrException(ErrorCode.BAD_REQUEST, "should have exactly one command");
+ op = commands.get(0);
+ }
+
+ fun.call(new ApiInfo(req, rsp, apiHandler, op));
+ if(op!= null && op.hasError()){
+ throw new ApiBag.ExceptionWithErrObject(ErrorCode.BAD_REQUEST,"error processing commands", captureErrors(singletonList(op)));
+ }
}
});
}
- static class ApiParams {
+ @Override
+ protected List<V2EndPoint> getEndPoints() {
+ return asList(CollectionApiMapping.EndPoint.values());
+ }
+
+ @Override
+ protected Collection<ApiCommand> getCommands() {
+ return apiCommands;
+ }
+
+ interface Command {
+
+ void call(ApiInfo info) throws Exception;
+
+ }
+
+ static class ApiInfo {
final SolrQueryRequest req;
final SolrQueryResponse rsp;
final BaseHandlerApiSupport apiHandler;
+ final CommandOperation op;
- ApiParams(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) {
+ ApiInfo(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler, CommandOperation op) {
this.req = req;
this.rsp = rsp;
this.apiHandler = apiHandler;
+ this.op = op;
}
}
- public CollectionHandlerApi(CollectionsHandler handler) {
- this.handler = handler;
- }
-
- @Override
- protected Collection<ApiCommand> getCommands() {
- return apiCommands;
- }
-
- @Override
- protected List<V2EndPoint> getEndPoints() {
- return Arrays.asList(CollectionApiMapping.EndPoint.values());
- }
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java b/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
index c082e37..c76bd3d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
@@ -17,17 +17,35 @@
package org.apache.solr.cloud;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.Utils;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.apache.solr.common.util.Utils.getObjectByPath;
+import static org.apache.solr.core.TestDynamicLoading.getFileContent;
+
public class TestClusterProperties extends SolrCloudTestCase {
private ClusterProperties props;
-
+
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(1).configure();
@@ -49,7 +67,119 @@ public class TestClusterProperties extends SolrCloudTestCase {
CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "false").process(cluster.getSolrClient());
assertEquals("false", props.getClusterProperty(ZkStateReader.LEGACY_CLOUD, "true"));
}
-
+
+ public void testSetClusterReqHandler() throws Exception {
+ new V2Request.Builder("/cluster")
+ .withPayload("{add-requesthandler:{name : '/foo', class : 'org.apache.solr.handler.DumpRequestHandler'}}")
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ Map<String, Object> map = new ClusterProperties(zkClient()).getClusterProperties();
+
+ assertEquals("org.apache.solr.handler.DumpRequestHandler",
+ getObjectByPath(map, true, Arrays.asList("requestHandler", "/foo", "class")));
+
+ new V2Request.Builder("/cluster")
+ .withPayload("{delete-requesthandler: '/foo'}")
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+
+ assertNull(getObjectByPath(map, true, Arrays.asList("requestHandler", "/foo")));
+
+ }
+
+ @Test
+ public void testRuntimeLib() throws Exception {
+ int port = getFreePort();
+ if (port == 0) {
+ fail("No port to be found");
+ }
+ Map<String, Object> jars = Utils.makeMap("/jar1.jar", getFileContent("runtimecode/runtimelibs.jar.bin"),
+ "/jar2.jar", getFileContent("runtimecode/runtimelibs_v2.jar.bin"));
+
+
+
+ Server server = null;
+ try {
+ server = new Server(port);
+ server.setHandler(new AbstractHandler() {
+ @Override
+ public void handle(String s, Request request, HttpServletRequest req, HttpServletResponse rsp)
+ throws IOException {
+ ByteBuffer b = (ByteBuffer) jars.get(s);
+ if (b != null) {
+ rsp.getOutputStream().write(b.array(), 0, b.limit());
+ rsp.setContentType("application/octet-stream");
+ rsp.setStatus(HttpServletResponse.SC_OK);
+ request.setHandled(true);
+ }
+ }
+ });
+ server.start();
+
+
+ String payload = null;
+
+ try {
+ payload = "{add-runtimelib:{name : 'foo', url: 'http://localhost:" + port + "/jar1.jar', " +
+ "sha512 : 'wrong-sha512'}}";
+ new V2Request.Builder("/cluster")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ fail("Expected error");
+ } catch (BaseHttpSolrClient.RemoteExecutionException e) {
+ assertTrue( e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("expected sha512 hash :"));
+ }
+
+ try {
+ payload = "{add-runtimelib:{name : 'foo', url: 'http://localhost:" + port + "/jar3.jar', " +
+ "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
+ new V2Request.Builder("/cluster")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ fail("Expected error");
+ } catch (BaseHttpSolrClient.RemoteExecutionException e) {
+ assertTrue( e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("no such resource available: foo"));
+ }
+
+ payload = "{add-runtimelib:{name : 'foo', url: 'http://localhost:" + port + "/jar1.jar', " +
+ "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
+ new V2Request.Builder("/cluster")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-runtimelib/sha512"),
+ getObjectByPath(new ClusterProperties(zkClient()).getClusterProperties(), true, "runtimeLib/foo/sha512"));
+
+ payload = "{update-runtimelib:{name : 'foo', url: 'http://localhost:" + port + "/jar2.jar', " +
+ "sha512 : 'bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417'}}";
+ new V2Request.Builder("/cluster")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-runtimelib/sha512"),
+ getObjectByPath(new ClusterProperties(zkClient()).getClusterProperties(), true, "runtimeLib/foo/sha512"));
+ } finally {
+ server.stop();
+ }
+ }
+
+ private int getFreePort() {
+ int port = 0;
+ int start = 30000 + random().nextInt(10000);
+ for (int i = 0; i < 10000; i++) {
+ try {
+ new ServerSocket(start + i).close();
+ port = 35000 + i;
+ break;
+ } catch (IOException e) {
+ continue;
+ }
+ }
+ return port;
+ }
+
@Test
public void testSetPluginClusterProperty() throws Exception {
String propertyName = ClusterProperties.EXT_PROPRTTY_PREFIX + "pluginA.propertyA";
@@ -57,7 +187,7 @@ public class TestClusterProperties extends SolrCloudTestCase {
.process(cluster.getSolrClient());
assertEquals("valueA", props.getClusterProperty(propertyName, null));
}
-
+
@Test(expected = SolrException.class)
public void testSetInvalidPluginClusterProperty() throws Exception {
String propertyName = "pluginA.propertyA";
diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
index 90d9afd..9a16602 100644
--- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
+++ b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
@@ -97,7 +97,7 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
assertNotNull(map = (Map) map.get("error"));
- assertTrue("full output " + map, map.get("msg").toString().contains("no such blob or version available: colltest/1" ));
+ assertTrue("full output " + map, map.get("msg").toString().contains("no such resource available: colltest/1" ));
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val',\n" +
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
index 74d0bbc..964c0a2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
@@ -207,6 +207,23 @@ public class CollectionApiMapping {
POST,
null,
"set-obj-property", null),
+
+ ADD_RUNTIME_LIB(CLUSTER_CMD,
+ POST,null,
+ "add-runtimelib",null ),
+ UPDATE_RUNTIME_LIB(CLUSTER_CMD,
+ POST,null,
+ "update-runtimelib",null ),
+ DELETE_RUNTIME_LIB(CLUSTER_CMD,
+ POST,null,
+ "delete-runtimelib",null ),
+ ADD_REQ_HANDLER(CLUSTER_CMD,
+ POST,null,
+ "add-requesthandler",null ),
+ DELETE_REQ_HANDLER(CLUSTER_CMD,
+ POST,null,
+ "delete-requesthandler",null ),
+
UTILIZE_NODE(CLUSTER_CMD,
POST,
UTILIZENODE,
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
index 96e5371..8c53cdc 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
@@ -105,6 +105,12 @@ public class ClusterProperties {
}
}
+ /**This applies the new map over the existing map. it's a merge operation, not an overwrite
+ * This applies the changes atomically over an existing object tree even if multiple nodes are
+ * trying to update this simultaneously
+ *
+ * @param properties The partial Object tree that needs to be applied
+ */
public void setClusterProperties(Map<String, Object> properties) throws IOException, KeeperException, InterruptedException {
client.atomicUpdate(ZkStateReader.CLUSTER_PROPS, zkData -> {
if (zkData == null) return Utils.toJSON(convertCollectionDefaultsToNestedFormat(properties));
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 88b67ff..69d565b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -277,7 +277,10 @@ public class ZkStateReader implements SolrCloseable {
DEFAULT_SHARD_PREFERENCES,
MAX_CORES_PER_NODE,
SAMPLE_PERCENTAGE,
- CollectionAdminParams.DEFAULTS)));
+ CollectionAdminParams.DEFAULTS,
+ "requestHandler",
+ "runtimeLib"
+ )));
/**
* Returns config set name for collection.
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 4a8d987..4eb3a3c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -91,7 +91,7 @@ public class Utils {
public static final Function NEW_SYNCHRONIZED_ARRAYLIST_FUN = o -> Collections.synchronizedList(new ArrayList<>());
public static final Function NEW_HASHSET_FUN = o -> new HashSet<>();
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+
public static Map getDeepCopy(Map map, int maxDepth) {
return getDeepCopy(map, maxDepth, true, false);
}
@@ -100,21 +100,18 @@ public class Utils {
return getDeepCopy(map, maxDepth, mutable, false);
}
- public static Map getDeepCopy(Map map, int maxDepth, boolean mutable, boolean sorted) {
- if(map == null) return null;
- if (maxDepth < 1) return map;
- Map copy;
- if (sorted) {
- copy = new TreeMap();
- } else {
- copy = map instanceof LinkedHashMap? new LinkedHashMap(map.size()): new HashMap(map.size());
- }
- for (Object o : map.entrySet()) {
- Map.Entry e = (Map.Entry) o;
- copy.put(e.getKey(), makeDeepCopy(e.getValue(),maxDepth, mutable, sorted));
+ public static final Function<JSONParser, ObjectBuilder> MAPWRITEROBJBUILDER = jsonParser -> {
+ try {
+ return new ObjectBuilder(jsonParser) {
+ @Override
+ public Object newObject() {
+ return new LinkedHashMapWriter();
+ }
+ };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- return mutable ? copy : Collections.unmodifiableMap(copy);
- }
+ };
public static void forEachMapEntry(Object o, String path, BiConsumer fun) {
Object val = Utils.getObjectByPath(o, false, path);
@@ -144,6 +141,40 @@ public class Utils {
((Map) o).forEach((k, v) -> fun.accept(k, v));
}
}
+ public static final Function<JSONParser, ObjectBuilder> MAPOBJBUILDER = jsonParser -> {
+ try {
+ return new ObjectBuilder(jsonParser) {
+ @Override
+ public Object newObject() {
+ return new HashMap();
+ }
+ };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
+ public static final Pattern ARRAY_ELEMENT_INDEX = Pattern
+ .compile("(\\S*?)\\[([-]?\\d+)\\]");
+
+ public static Map getDeepCopy(Map map, int maxDepth, boolean mutable, boolean sorted) {
+ if (map == null) return null;
+ if (maxDepth < 1) return map;
+ Map copy;
+ if (sorted) {
+ copy = new TreeMap();
+ } else {
+ copy = map instanceof LinkedHashMap ? new LinkedHashMap(map.size()) : new HashMap(map.size());
+ }
+ for (Object o : map.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ copy.put(e.getKey(), makeDeepCopy(e.getValue(), maxDepth, mutable, sorted));
+ }
+ return mutable ? copy : Collections.unmodifiableMap(copy);
+ }
+
+ public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable) {
+ return getDeepCopy(c, maxDepth, mutable, false);
+ }
private static Object makeDeepCopy(Object v, int maxDepth, boolean mutable, boolean sorted) {
if (v instanceof MapWriter && maxDepth > 1) {
@@ -151,7 +182,7 @@ public class Utils {
} else if (v instanceof IteratorWriter && maxDepth > 1) {
v = ((IteratorWriter) v).toList(new ArrayList<>());
if (sorted) {
- Collections.sort((List)v);
+ Collections.sort((List) v);
}
}
@@ -163,29 +194,6 @@ public class Utils {
return v;
}
- public static InputStream toJavabin(Object o) throws IOException {
- try (final JavaBinCodec jbc = new JavaBinCodec()) {
- BinaryRequestWriter.BAOS baos = new BinaryRequestWriter.BAOS();
- jbc.marshal(o,baos);
- return new ByteBufferInputStream(ByteBuffer.wrap(baos.getbuf(),0,baos.size()));
- }
- }
-
- public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable) {
- return getDeepCopy(c, maxDepth, mutable, false);
- }
-
- public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable, boolean sorted) {
- if (c == null || maxDepth < 1) return c;
- Collection result = c instanceof Set ?
- ( sorted? new TreeSet() : new HashSet()) : new ArrayList();
- for (Object o : c) result.add(makeDeepCopy(o, maxDepth, mutable, sorted));
- if (sorted && (result instanceof List)) {
- Collections.sort((List)result);
- }
- return mutable ? result : result instanceof Set ? unmodifiableSet((Set) result) : unmodifiableList((List) result);
- }
-
public static void writeJson(Object o, OutputStream os, boolean indent) throws IOException {
writeJson(o, new OutputStreamWriter(os, UTF_8), indent)
.flush();
@@ -199,35 +207,12 @@ public class Utils {
return writer;
}
- private static class MapWriterJSONWriter extends JSONWriter {
-
- public MapWriterJSONWriter(CharArr out, int indentSize) {
- super(out, indentSize);
- }
-
- @Override
- public void handleUnknownClass(Object o) {
- if (o instanceof MapWriter) {
- Map m = ((MapWriter)o).toMap(new LinkedHashMap<>());
- write(m);
- } else {
- super.handleUnknownClass(o);
- }
- }
- }
-
- public static byte[] toJSON(Object o) {
- if(o == null) return new byte[0];
- CharArr out = new CharArr();
- if (!(o instanceof List) && !(o instanceof Map)) {
- if (o instanceof MapWriter) {
- o = ((MapWriter)o).toMap(new LinkedHashMap<>());
- } else if(o instanceof IteratorWriter){
- o = ((IteratorWriter)o).toList(new ArrayList<>());
- }
+ public static InputStream toJavabin(Object o) throws IOException {
+ try (final JavaBinCodec jbc = new JavaBinCodec()) {
+ BinaryRequestWriter.BAOS baos = new BinaryRequestWriter.BAOS();
+ jbc.marshal(o, baos);
+ return new ByteBufferInputStream(ByteBuffer.wrap(baos.getbuf(), 0, baos.size()));
}
- new MapWriterJSONWriter(out, 2).write(o); // indentation by default
- return toUTF8(out);
}
public static String toJSONString(Object o) {
@@ -274,15 +259,29 @@ public class Utils {
return propMap;
}
- public static Object fromJSON(InputStream is){
- return fromJSON(new InputStreamReader(is, UTF_8));
+ public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable, boolean sorted) {
+ if (c == null || maxDepth < 1) return c;
+ Collection result = c instanceof Set ?
+ (sorted ? new TreeSet() : new HashSet()) : new ArrayList();
+ for (Object o : c) result.add(makeDeepCopy(o, maxDepth, mutable, sorted));
+ if (sorted && (result instanceof List)) {
+ Collections.sort((List) result);
+ }
+ return mutable ? result : result instanceof Set ? unmodifiableSet((Set) result) : unmodifiableList((List) result);
}
- public static Object fromJSON(Reader is){
- try {
- return STANDARDOBJBUILDER.apply(getJSONParser(is)).getVal();
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error", e);
+
+ public static byte[] toJSON(Object o) {
+ if (o == null) return new byte[0];
+ CharArr out = new CharArr();
+ if (!(o instanceof List) && !(o instanceof Map)) {
+ if (o instanceof MapWriter) {
+ o = ((MapWriter) o).toMap(new LinkedHashMap<>());
+ } else if (o instanceof IteratorWriter) {
+ o = ((IteratorWriter) o).toList(new ArrayList<>());
+ }
}
+ new MapWriterJSONWriter(out, 2).write(o); // indentation by default
+ return toUTF8(out);
}
@@ -293,35 +292,14 @@ public class Utils {
throw new RuntimeException(e);
}
};
- public static final Function<JSONParser, ObjectBuilder> MAPWRITEROBJBUILDER = jsonParser -> {
- try {
- return new ObjectBuilder(jsonParser){
- @Override
- public Object newObject() {
- return new LinkedHashMapWriter();
- }
- };
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- };
- public static final Function<JSONParser, ObjectBuilder> MAPOBJBUILDER = jsonParser -> {
- try {
- return new ObjectBuilder(jsonParser){
- @Override
- public Object newObject() {
- return new HashMap();
- }
- };
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- };
+ public static Object fromJSON(InputStream is) {
+ return fromJSON(new InputStreamReader(is, UTF_8));
+ }
- public static Object fromJSON(InputStream is, Function<JSONParser, ObjectBuilder> objBuilderProvider) {
+ public static Object fromJSON(Reader is) {
try {
- return objBuilderProvider.apply(getJSONParser((new InputStreamReader(is, StandardCharsets.UTF_8)))).getVal();
+ return STANDARDOBJBUILDER.apply(getJSONParser(is)).getVal();
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error", e);
}
@@ -336,10 +314,19 @@ public class Utils {
return fromJSON(stream);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Resource error: " + e.getMessage(), e);
+ "Resource error: " + e.getMessage(), e);
}
}
- public static JSONParser getJSONParser(Reader reader){
+
+ public static Object fromJSON(InputStream is, Function<JSONParser, ObjectBuilder> objBuilderProvider) {
+ try {
+ return objBuilderProvider.apply(getJSONParser((new InputStreamReader(is, StandardCharsets.UTF_8)))).getVal();
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error", e);
+ }
+ }
+
+ public static JSONParser getJSONParser(Reader reader) {
JSONParser parser = new JSONParser(reader);
parser.setFlags(parser.getFlags() |
JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT |
@@ -347,11 +334,11 @@ public class Utils {
return parser;
}
- public static Object fromJSONString(String json) {
+ public static Object fromJSONString(String json) {
try {
return STANDARDOBJBUILDER.apply(getJSONParser(new StringReader(json))).getVal();
} catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error : "+ json, e );
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error : " + json, e);
}
}
@@ -363,10 +350,10 @@ public class Utils {
public static boolean setObjectByPath(Object root, String hierarchy, Object value) {
List<String> parts = StrUtils.splitSmart(hierarchy, '/', true);
- return setObjectByPath(root, parts, value);
+ return setObjectByPath(root, parts, value, true);
}
- public static boolean setObjectByPath(Object root, List<String> hierarchy, Object value) {
+ public static boolean setObjectByPath(Object root, List<String> hierarchy, Object value, boolean insertMissing) {
if (root == null) return false;
if (!isMapLike(root)) throw new RuntimeException("must be a Map or NamedList");
Object obj = root;
@@ -382,7 +369,10 @@ public class Utils {
}
if (i < hierarchy.size() - 1) {
Object o = getVal(obj, s, -1);
- if (o == null) return false;
+ if (o == null) {
+ if (insertMissing) insertItem(o = new LinkedHashMap<>(), obj, s);
+ else return false;
+ }
if (idx > -1) {
List l = (List) o;
o = idx < l.size() ? l.get(idx) : null;
@@ -391,14 +381,7 @@ public class Utils {
obj = o;
} else {
if (idx == -2) {
- if (obj instanceof NamedList) {
- NamedList namedList = (NamedList) obj;
- int location = namedList.indexOf(s, 0);
- if (location == -1) namedList.add(s, value);
- else namedList.setVal(location, value);
- } else if (obj instanceof Map) {
- ((Map) obj).put(s, value);
- }
+ insertItem(value, obj, s);
return true;
} else {
Object v = getVal(obj, s, -1);
@@ -422,10 +405,20 @@ public class Utils {
}
+ private static void insertItem(Object value, Object container, String name) {
+ if (container instanceof NamedList) {
+ NamedList namedList = (NamedList) container;
+ int location = namedList.indexOf(name, 0);
+ if (location == -1) namedList.add(name, value);
+ else namedList.setVal(location, value);
+ } else if (container instanceof Map) {
+ ((Map) container).put(name, value);
+ }
+ }
public static Object getObjectByPath(Object root, boolean onlyPrimitive, List<String> hierarchy) {
- if(root == null) return null;
- if(!isMapLike(root)) return null;
+ if (root == null) return null;
+ if (!isMapLike(root)) return null;
Object obj = root;
for (int i = 0; i < hierarchy.size(); i++) {
int idx = -1;
@@ -518,6 +511,7 @@ public class Utils {
try {
((MapWriter) obj).writeMap(new MapWriter.EntryWriter() {
int count = -1;
+
@Override
public MapWriter.EntryWriter put(CharSequence k, Object v) {
if (result[0] != null) return this;
@@ -533,15 +527,14 @@ public class Utils {
throw new RuntimeException(e);
}
return result[0];
- }
- else if (obj instanceof Map) return ((Map) obj).get(key);
+ } else if (obj instanceof Map) return ((Map) obj).get(key);
else throw new RuntimeException("must be a NamedList or Map");
}
/**
* If the passed entity has content, make sure it is fully
* read and closed.
- *
+ *
* @param entity to consume or null
*/
public static void consumeFully(HttpEntity entity) {
@@ -562,31 +555,21 @@ public class Utils {
/**
* Make sure the InputStream is fully read.
- *
+ *
* @param is to read
* @throws IOException on problem with IO
*/
private static void readFully(InputStream is) throws IOException {
is.skip(is.available());
- while (is.read() != -1) {}
- }
-
- public static Map<String, Object> getJson(DistribStateManager distribStateManager, String path) throws InterruptedException, IOException, KeeperException {
- VersionedData data = null;
- try {
- data = distribStateManager.getData(path);
- } catch (KeeperException.NoNodeException | NoSuchElementException e) {
- return Collections.emptyMap();
+ while (is.read() != -1) {
}
- if (data == null || data.getData() == null || data.getData().length == 0) return Collections.emptyMap();
- return (Map<String, Object>) Utils.fromJSON(data.getData());
}
/**
* Assumes data in ZooKeeper is a JSON string, deserializes it and returns as a Map
*
- * @param zkClient the zookeeper client
- * @param path the path to the znode being read
+ * @param zkClient the zookeeper client
+ * @param path the path to the znode being read
* @param retryOnConnLoss whether to retry the operation automatically on connection loss, see {@link org.apache.solr.common.cloud.ZkCmdExecutor#retryOperation(ZkOperation)}
* @return a Map if the node exists and contains valid JSON or an empty map if znode does not exist or has a null data
*/
@@ -602,39 +585,23 @@ public class Utils {
return Collections.emptyMap();
}
- public static final Pattern ARRAY_ELEMENT_INDEX = Pattern
- .compile("(\\S*?)\\[([-]?\\d+)\\]");
-
- public static SpecProvider getSpec(final String name) {
- return () -> {
- return ValidatingJsonMap.parse(CommonParams.APISPEC_LOCATION + name + ".json", CommonParams.APISPEC_LOCATION);
- };
- }
-
- public static String parseMetricsReplicaName(String collectionName, String coreName) {
- if (collectionName == null || !coreName.startsWith(collectionName)) {
- return null;
- } else {
- // split "collection1_shard1_1_replica1" into parts
- if (coreName.length() > collectionName.length()) {
- String str = coreName.substring(collectionName.length() + 1);
- int pos = str.lastIndexOf("_replica");
- if (pos == -1) { // ?? no _replicaN part ??
- return str;
- } else {
- return str.substring(pos + 1);
- }
- } else {
- return null;
- }
+ public static Map<String, Object> getJson(DistribStateManager distribStateManager, String path) throws InterruptedException, IOException, KeeperException {
+ VersionedData data = null;
+ try {
+ data = distribStateManager.getData(path);
+ } catch (KeeperException.NoNodeException | NoSuchElementException e) {
+ return Collections.emptyMap();
}
+ if (data == null || data.getData() == null || data.getData().length == 0) return Collections.emptyMap();
+ return (Map<String, Object>) Utils.fromJSON(data.getData());
}
- /**Applies one json over other. The 'input' is applied over the sink
- * The values in input isapplied over the values in 'sink' . If a value is 'null'
+ /**
+ * Applies one json over other. The 'input' is applied over the sink
+ * The values in input are applied over the values in 'sink' . If a value is 'null'
* that value is removed from sink
*
- * @param sink the original json object to start with. Ensure that this Map is mutable
+ * @param sink the original json object to start with. Ensure that this Map is mutable
* @param input the json with new values
* @return whether there was any change made to sink or not.
*/
@@ -672,20 +639,62 @@ public class Utils {
return isModified;
}
+ public static SpecProvider getSpec(final String name) {
+ return () -> {
+ return ValidatingJsonMap.parse(CommonParams.APISPEC_LOCATION + name + ".json", CommonParams.APISPEC_LOCATION);
+ };
+ }
+
+ public static String parseMetricsReplicaName(String collectionName, String coreName) {
+ if (collectionName == null || !coreName.startsWith(collectionName)) {
+ return null;
+ } else {
+ // split "collection1_shard1_1_replica1" into parts
+ if (coreName.length() > collectionName.length()) {
+ String str = coreName.substring(collectionName.length() + 1);
+ int pos = str.lastIndexOf("_replica");
+ if (pos == -1) { // ?? no _replicaN part ??
+ return str;
+ } else {
+ return str.substring(pos + 1);
+ }
+ } else {
+ return null;
+ }
+ }
+ }
+
public static String getBaseUrlForNodeName(final String nodeName, String urlScheme) {
final int _offset = nodeName.indexOf("_");
if (_offset < 0) {
throw new IllegalArgumentException("nodeName does not contain expected '_' separator: " + nodeName);
}
- final String hostAndPort = nodeName.substring(0,_offset);
+ final String hostAndPort = nodeName.substring(0, _offset);
try {
- final String path = URLDecoder.decode(nodeName.substring(1+_offset), "UTF-8");
+ final String path = URLDecoder.decode(nodeName.substring(1 + _offset), "UTF-8");
return urlScheme + "://" + hostAndPort + (path.isEmpty() ? "" : ("/" + path));
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
}
}
+ private static class MapWriterJSONWriter extends JSONWriter {
+
+ public MapWriterJSONWriter(CharArr out, int indentSize) {
+ super(out, indentSize);
+ }
+
+ @Override
+ public void handleUnknownClass(Object o) {
+ if (o instanceof MapWriter) {
+ Map m = ((MapWriter) o).toMap(new LinkedHashMap<>());
+ write(m);
+ } else {
+ super.handleUnknownClass(o);
+ }
+ }
+ }
+
public static long time(TimeSource timeSource, TimeUnit unit) {
return unit.convert(timeSource.getTimeNs(), TimeUnit.NANOSECONDS);
}
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json
index 069cd1d..0b4f33f 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.json
@@ -169,6 +169,43 @@
"required": [
"name"
]
+ },
+ "add-runtimelib": {
+ "documentation": "",
+ "description" : "Add a remote jar to the classpath",
+ "#include": "cluster.Commands.runtimelib.properties"
+ },
+ "update-runtimelib": {
+ "documentation": "",
+ "description" : "Update the jar details",
+ "#include": "cluster.Commands.runtimelib.properties"
+ },
+ "delete-runtimelib": {
+ "documentation": "",
+ "description" : "delete a lib",
+ "type": "string"
+ },
+ "add-requesthandler": {
+ "type": "object",
+ "documentation": "",
+ "description" : "Create a node level request handler",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the request handler. This is the path"
+ },
+ "class": {
+ "type": "string",
+ "description": "The class name"
+ }
+ },
+ "required": ["name", "class"],
+ "additionalProperties": true
+ },
+ "delete-requesthandler" : {
+ "description" : "delete a requesthandler",
+ "type": "string"
}
+
}
}
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json b/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json
new file mode 100644
index 0000000..5a8a66f
--- /dev/null
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json
@@ -0,0 +1,23 @@
+{
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "A name for the library"
+ },
+ "url": {
+ "type": "string",
+ "description": "The remote url"
+ },
+ "sha512": {
+ "type": "string",
+ "description": "The sha512 hash of the jar"
+ },
+ "sig": {
+ "type": "string",
+ "description": "the signature of the jar"
+ }
+ },
+ "required" : ["name","url","sha512"]
+
+}
\ No newline at end of file