You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by is...@apache.org on 2022/02/02 11:35:10 UTC
[solr] 01/01: Local packages
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-15951
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 23b7443c2a89f6c5cffa9421f22a41a1862aabb9
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Wed Feb 2 17:04:52 2022 +0530
Local packages
---
gradle/solr/packaging.gradle | 2 +
solr/bin/solr | 1 +
.../solr/client/solrj/embedded/JettyConfig.java | 15 ++-
.../src/java/org/apache/solr/pkg/PackageAPI.java | 8 +-
.../java/org/apache/solr/pkg/PackageLoader.java | 123 ++++++++++++++---
.../apache/solr/cloud/PackageManagerCLITest.java | 5 +-
.../cluster/events/ClusterEventProducerTest.java | 5 +-
.../solr/filestore/TestDistribPackageStore.java | 5 +-
.../apache/solr/handler/TestContainerPlugin.java | 5 +-
.../org/apache/solr/pkg/TestLocalPackages.java | 146 +++++++++++++++++++++
.../src/test/org/apache/solr/pkg/TestPackages.java | 8 +-
solr/modules/build.gradle | 71 ++++++++++
solr/modules/extraction/manifest.json | 22 ++++
solr/modules/scripting/manifest.json | 23 ++++
.../update/ScriptUpdateProcessorFactory.java | 7 +-
.../apache/solr/cloud/MiniSolrCloudCluster.java | 1 +
16 files changed, 411 insertions(+), 36 deletions(-)
diff --git a/gradle/solr/packaging.gradle b/gradle/solr/packaging.gradle
index e0bd891..7a33582 100644
--- a/gradle/solr/packaging.gradle
+++ b/gradle/solr/packaging.gradle
@@ -65,6 +65,8 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules
task assemblePackaging(type: Sync) {
from "README.md"
+ from "manifest.json"
+
from (tasks.jar, {
into "lib"
})
diff --git a/solr/bin/solr b/solr/bin/solr
index 3a6f0e4..3eb5c87 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -2267,6 +2267,7 @@ function start_solr() {
# users who don't care about useful error msgs can override in SOLR_OPTS with +OmitStackTraceInFastThrow
"${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" "-XX:-OmitStackTraceInFastThrow" \
"-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \
+ "-Dsolr.packages.local.dir=$SOLR_TIP/modules" \
"-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.data.home=$SOLR_DATA_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
"-Dsolr.default.confdir=$DEFAULT_CONFDIR" "${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}" "${SECURITY_MANAGER_OPTS[@]}" "${SOLR_ADMIN_UI}")
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
index e4a0547..0f97f18 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
@@ -22,6 +22,7 @@ import javax.servlet.Filter;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeMap;
+import java.util.function.Consumer;
public class JettyConfig {
@@ -47,9 +48,12 @@ public class JettyConfig {
public final int portRetryTime;
+ public final Consumer<JettySolrRunner> preStartupHook;
+
private JettyConfig(boolean onlyHttp1, int port, int portRetryTime , String context, boolean stopAtShutdown,
Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets,
- Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig, boolean enableV2) {
+ Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig, boolean enableV2,
+ Consumer<JettySolrRunner> preStartupHook) {
this.onlyHttp1 = onlyHttp1;
this.port = port;
this.context = context;
@@ -60,6 +64,7 @@ public class JettyConfig {
this.sslConfig = sslConfig;
this.portRetryTime = portRetryTime;
this.enableV2 = enableV2;
+ this.preStartupHook = preStartupHook;
}
public static Builder builder() {
@@ -89,6 +94,7 @@ public class JettyConfig {
Map<Class<? extends Filter>, String> extraFilters = new LinkedHashMap<>();
SSLConfig sslConfig = null;
int portRetryTime = 60;
+ Consumer<JettySolrRunner> preStartupHook;
public Builder useOnlyHttp1(boolean useOnlyHttp1) {
this.onlyHttp1 = useOnlyHttp1;
@@ -151,10 +157,15 @@ public class JettyConfig {
return this;
}
+ public Builder withPreStartupHook(Consumer<JettySolrRunner> preStartupHook) {
+ this.preStartupHook = preStartupHook;
+ return this;
+ }
+
public JettyConfig build() {
return new JettyConfig(onlyHttp1, port, portRetryTime, context, stopAtShutdown,
- waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig, enableV2);
+ waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig, enableV2, preStartupHook);
}
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index 3a01c1b..1c197b4 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -60,13 +60,12 @@ import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_
*
*/
public class PackageAPI {
- public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false"));
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String ERR_MSG = "Package loading is not enabled , Start your nodes with -Denable.packages=true";
final CoreContainer coreContainer;
- private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
+ public static final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper();
private final PackageLoader packageLoader;
Packages pkgs;
@@ -81,6 +80,7 @@ public class PackageAPI {
this.coreContainer = coreContainer;
this.packageLoader = loader;
pkgs = new Packages();
+ if(!loader.repoPackagesEnabled) return;
SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
try {
pkgs = readPkgsFromZk(null, null);
@@ -360,11 +360,11 @@ public class PackageAPI {
}
public boolean isEnabled() {
- return enablePackages;
+ return packageLoader.repoPackagesEnabled;
}
private boolean checkEnabled(CommandOperation payload) {
- if (!enablePackages) {
+ if (!packageLoader.repoPackagesEnabled) {
payload.addError(ERR_MSG);
return false;
}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
index b73be8c..bb85eb8 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
@@ -17,10 +17,9 @@
package org.apache.solr.pkg;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -37,7 +36,9 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
@@ -52,14 +53,24 @@ import static org.apache.lucene.util.IOUtils.closeWhileHandlingException;
public class PackageLoader implements Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String LATEST = "$LATEST";
+ public static final String LOCAL_PKGS_DIR_PROP = "solr.packages.local.dir";
+ public static final String ENABLED_LOCAL_PKGS_PROP = "solr.enabled.local.pkgs";
+ public static final String LOCAL_PACKAGES_JSON = "local_packages.json";
+ public static final String ENABLE_PACKAGES_REPO_PROP = "enable.packages";
+ public static final String ENABLE_PACKAGES_REPO_PROP_NEW = "solr.enable.pkgs.repo";
+ public final String localPkgsDir = System.getProperty(LOCAL_PKGS_DIR_PROP);
+ public final String enabledLocalPkgsList = System.getProperty(ENABLED_LOCAL_PKGS_PROP, "");
+ public final boolean repoPackagesEnabled = Boolean.parseBoolean(System.getProperty(ENABLE_PACKAGES_REPO_PROP,
+ System.getProperty(ENABLE_PACKAGES_REPO_PROP_NEW, "false")));
private final CoreContainer coreContainer;
private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
+ public PackageAPI.Packages localPackages;
private PackageAPI.Packages myCopy = new PackageAPI.Packages();
- private PackageAPI packageAPI;
+ private final PackageAPI packageAPI;
public Optional<Package.Version> getPackageVersion(String pkg, String version) {
@@ -70,9 +81,80 @@ public class PackageLoader implements Closeable {
public PackageLoader(CoreContainer coreContainer) {
this.coreContainer = coreContainer;
+
+ List<String> enabledPackages = StrUtils.splitSmart(enabledLocalPkgsList, ',');
packageAPI = new PackageAPI(coreContainer, this);
- refreshPackageConf();
+ if (localPkgsDir != null && !enabledPackages.isEmpty()) {
+ loadLocalPackages(localPkgsDir, enabledPackages);
+ }
+ if (repoPackagesEnabled) {
+ refreshPackageConf();
+ }
+ }
+
+ private void loadLocalPackages(String localPkgsDir, List<String> enabledPackages) {
+ final Path packagesPath = localPkgsDir.charAt(0) == File.separatorChar ?
+ Paths.get(localPkgsDir) :
+ Paths.get(coreContainer.getSolrHome()).resolve(localPkgsDir);
+ log.info("Packages to be loaded from directory: {}", packagesPath);
+
+ if (!Files.exists(packagesPath)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No such directory: " + packagesPath);
+ }
+ Path packagesJsonPath = packagesPath.resolve(LOCAL_PACKAGES_JSON);
+ if(Files.exists(packagesJsonPath)) {
+ try {
+
+ try (InputStream in = Files.newInputStream(packagesJsonPath)) {
+ localPackages = PackageAPI.mapper.readValue(in, PackageAPI.Packages.class);
+ }
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading local_packages.json", e);
+ }
+ } else {
+ //the no local_packages.json
+ //we will look for each subdirectory and consider them as a package
+ localPackages = readDirAsPackages(packagesPath);
+ }
+
+ for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : localPackages.packages.entrySet()) {
+ if(!enabledPackages.contains(e.getKey())) continue;
+ Package p = new Package(e.getKey());
+ p.updateVersions(e.getValue(), packagesPath);
+ packageClassLoaders.put(e.getKey(), p);
+ }
+ }
+
+ /**
+ * If a directory with no local_packages.json is provided,
+ * each sub directory that contains one or more jar files
+ * will be treated as a package and each jar file in that
+ * directory is added to the package classpath
+ */
+ private PackageAPI.Packages readDirAsPackages(Path packagesPath) {
+ PackageAPI.Packages localDirAsPackages = new PackageAPI.Packages();
+ packagesPath.toFile().list((dir, pkgName) -> {
+ File subDir = new File(dir, pkgName);
+ if(subDir.isDirectory()) {
+ PackageAPI.PkgVersion version = new PackageAPI.PkgVersion();
+ version.pkg = pkgName;
+ version.version = "0";
+ version.files = new ArrayList<>();
+ subDir.list((dir1, jarName) -> {
+ if(jarName.endsWith(".jar")){
+ version.files.add(pkgName+File.separator+jarName);
+ }
+ return false;
+ });
+ if(!version.files.isEmpty()) {
+ //there are jar files in the dir. So, this is a package
+ localDirAsPackages.packages.put(pkgName, Collections.singletonList(version));
+ }
+ }
+ return false;
+ });
+ return localDirAsPackages;
}
public PackageAPI getPackageAPI() {
@@ -99,7 +181,7 @@ public class PackageLoader implements Closeable {
if (e.getValue() != null && p == null) {
packageClassLoaders.put(e.getKey(), p = new Package(e.getKey()));
}
- p.updateVersions(e.getValue());
+ p.updateVersions(e.getValue(), null);
updated.add(p);
} else {
Package p = packageClassLoaders.remove(e.getKey());
@@ -181,14 +263,14 @@ public class PackageLoader implements Closeable {
}
- private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified) {
+ private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified, Path localpkgDir) {
for (PackageAPI.PkgVersion v : modified) {
Version version = myVersions.get(v.version);
if (version == null) {
log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files);
Version ver = null;
try {
- ver = new Version(this, v);
+ ver = new Version(this, v, localpkgDir);
} catch (Exception e) {
log.error("package could not be loaded {}", ver, e);
continue;
@@ -274,20 +356,25 @@ public class PackageLoader implements Closeable {
version.writeMap(ew);
}
- Version(Package parent, PackageAPI.PkgVersion v) {
+ Version(Package parent, PackageAPI.PkgVersion v, Path localPkgDir) {
this.parent = parent;
this.version = v;
List<Path> paths = new ArrayList<>();
-
- List<String> errs = new ArrayList<>();
- coreContainer.getPackageStoreAPI().validateFiles(version.files, true, s -> errs.add(s));
- if(!errs.isEmpty()) {
- throw new RuntimeException("Cannot load package: " +errs);
- }
- for (String file : version.files) {
- paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
+ if(localPkgDir != null) {
+ for (String file : v.files) {
+ if(file.charAt(0)== '/') file =file.substring(1);
+ paths.add( localPkgDir.resolve(file).toAbsolutePath()) ;
+ }
+ } else {
+ List<String> errs = new ArrayList<>();
+ coreContainer.getPackageStoreAPI().validateFiles(version.files, true, s -> errs.add(s));
+ if(!errs.isEmpty()) {
+ throw new RuntimeException("Cannot load package: " +errs);
+ }
+ for (String file : version.files) {
+ paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
+ }
}
-
loader = new PackageResourceLoader(
"PACKAGE_LOADER: " + parent.name() + ":" + version,
paths,
diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
index aad9286..cf737a7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.core.TestSolrConfigHandler;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.util.LogLevel;
import org.apache.solr.util.PackageTool;
import org.apache.solr.util.SolrCLI;
@@ -51,7 +52,7 @@ public class PackageManagerCLITest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
- System.setProperty("enable.packages", "true");
+ System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true");
configureCluster(1)
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
@@ -65,7 +66,7 @@ public class PackageManagerCLITest extends SolrCloudTestCase {
@AfterClass
public static void teardown() throws Exception {
repositoryServer.stop();
- System.clearProperty("enable.packages");
+ System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP);
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java b/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java
index a96e209..4426806 100644
--- a/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java
@@ -29,6 +29,7 @@ import org.apache.solr.cluster.events.impl.DefaultClusterEventProducer;
import org.apache.solr.cluster.events.impl.DelegatingClusterEventProducer;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.util.LogLevel;
import org.junit.After;
import org.junit.Before;
@@ -70,7 +71,7 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
@Before
public void setUp() throws Exception {
- System.setProperty("enable.packages", "true");
+ System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true");
super.setUp();
cluster.deleteAllCollections();
eventsListener = new AllEventsListener();
@@ -85,7 +86,7 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
@After
public void teardown() throws Exception {
- System.clearProperty("enable.packages");
+ System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP);
if (eventsListener != null) {
cluster.getOpenOverseer().getCoreContainer().getClusterEventProducer().unregisterListener(eventsListener);
eventsListener.events.clear();
diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
index e25be64..711cb52 100644
--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java
@@ -36,6 +36,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.packagemanager.PackageUtils;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.util.LogLevel;
import org.apache.zookeeper.server.ByteBufferInputStream;
import org.junit.After;
@@ -63,12 +64,12 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
@Before
public void setup() {
- System.setProperty("enable.packages", "true");
+ System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true");
}
@After
public void teardown() {
- System.clearProperty("enable.packages");
+ System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP);
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java
index 7254b0f..a64de90 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java
@@ -52,6 +52,7 @@ import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.filestore.PackageStoreAPI;
import org.apache.solr.filestore.TestDistribPackageStore;
import org.apache.solr.filestore.TestDistribPackageStore.Fetcher;
+import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.pkg.TestPackages;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -71,13 +72,13 @@ public class TestContainerPlugin extends SolrCloudTestCase {
@Before
public void setup() {
- System.setProperty("enable.packages", "true");
+ System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true");
phaser = new Phaser();
}
@After
public void teardown() {
- System.clearProperty("enable.packages");
+ System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP);
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestLocalPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestLocalPackages.java
new file mode 100644
index 0000000..1dfe51f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/pkg/TestLocalPackages.java
@@ -0,0 +1,146 @@
+package org.apache.solr.pkg;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.pkg.PackageLoader.ENABLED_LOCAL_PKGS_PROP;
+import static org.apache.solr.pkg.PackageLoader.LOCAL_PKGS_DIR_PROP;
+
+public class TestLocalPackages extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public void testLocalPackagesAsDir() throws Exception {
+ String PKG_NAME = "mypkg";
+ String jarName = "mypkg1.jar";
+ String COLLECTION_NAME = "testLocalPkgsColl";
+ String localPackagesDir = "testpkgdir";
+ System.setProperty(ENABLED_LOCAL_PKGS_PROP, PKG_NAME);
+ System.setProperty(LOCAL_PKGS_DIR_PROP, localPackagesDir);
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(builder -> builder.enableV2(true))
+ .withJettyConfig(it -> it.withPreStartupHook(jsr -> {
+ try {
+ File pkgDir = new File(jsr.getSolrHome() + File.separator + localPackagesDir);
+ if (!pkgDir.exists()) {
+ pkgDir.mkdir();
+ }
+ File subDir = new File(pkgDir, PKG_NAME);
+ if (!subDir.exists()) {
+ subDir.mkdir();
+ }
+ try (FileInputStream fis = new FileInputStream(getFile("runtimecode/runtimelibs.jar.bin"))) {
+ byte[] buf = new byte[fis.available()];
+
+ fis.read(buf);
+ try (FileOutputStream fos = new FileOutputStream(new File(subDir, jarName))) {
+ fos.write(buf, 0, buf.length);
+ }
+ }
+
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to create files", e);
+ }
+ }))
+ .addConfig("conf", configset("conf2"))
+ .configure();
+
+ System.clearProperty(ENABLED_LOCAL_PKGS_PROP);
+ System.clearProperty(LOCAL_PKGS_DIR_PROP);
+ try {
+ for (JettySolrRunner jsr : cluster.getJettySolrRunners()) {
+ List<String> packageFiles = Arrays.asList(new File(jsr.getSolrHome() + File.separator + localPackagesDir + File.separator + PKG_NAME).list());
+ assertTrue(packageFiles.contains(jarName));
+ }
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 2, 2)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
+
+ log.info("Collection created successfully");
+
+ TestPackages.verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", PKG_NAME, "0");
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ public void testLocalPackages() throws Exception {
+ String PKG_NAME = "mypkg";
+ String jarName = "mypkg1.jar";
+ String COLLECTION_NAME = "testLocalPkgsColl";
+ String localPackagesDir = "local_packages";
+ PackageAPI.Packages p = new PackageAPI.Packages();
+ PackageAPI.PkgVersion pkgVersion = new PackageAPI.PkgVersion();
+ pkgVersion.files = Collections.singletonList(jarName);
+ pkgVersion.version = "0.1";
+ pkgVersion.pkg = PKG_NAME;
+ p.packages.put(PKG_NAME, Collections.singletonList(pkgVersion));
+
+ log.info("local_packages.json: {}" , Utils.toJSONString(p));
+ log.info("Local packages dir: {}" , localPackagesDir);
+ System.setProperty(ENABLED_LOCAL_PKGS_PROP, PKG_NAME);
+ System.setProperty(LOCAL_PKGS_DIR_PROP, localPackagesDir);
+ MiniSolrCloudCluster cluster =
+ configureCluster(4)
+ .withJettyConfig(builder -> builder.enableV2(true))
+ .withJettyConfig(it -> it.withPreStartupHook(jsr -> {
+ try {
+ File pkgDir = new File(jsr.getSolrHome() + File.separator + localPackagesDir);
+ if (!pkgDir.exists()) {
+ pkgDir.mkdir();
+ }
+ try (FileInputStream fis = new FileInputStream(getFile("runtimecode/runtimelibs.jar.bin"))) {
+ byte[] buf = new byte[fis.available()];
+
+ fis.read(buf);
+ try (FileOutputStream fos = new FileOutputStream(new File(pkgDir, jarName))) {
+ fos.write(buf, 0, buf.length);
+ }
+ }
+
+ try( FileOutputStream fos = new FileOutputStream( new File(pkgDir, PackageLoader.LOCAL_PACKAGES_JSON) )) {
+ fos.write(Utils.toJSON(p));
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to create files", e);
+ }
+ }))
+ .addConfig("conf", configset("conf2"))
+ .configure();
+
+ System.clearProperty(ENABLED_LOCAL_PKGS_PROP);
+ System.clearProperty(LOCAL_PKGS_DIR_PROP);
+ try {
+ for (JettySolrRunner jsr : cluster.getJettySolrRunners()) {
+ List<String> packageFiles = Arrays.asList(new File(jsr.getSolrHome() + File.separator + localPackagesDir).list());
+ assertTrue(packageFiles.contains(PackageLoader.LOCAL_PACKAGES_JSON));
+ assertTrue(packageFiles.contains(jarName));
+ }
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 2, 2)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
+
+ log.info("Collection created successfully");
+
+ TestPackages.verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", PKG_NAME, pkgVersion.version);
+
+
+ } finally {
+ cluster.shutdown();
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 4c91daa..b1d3b3b 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -82,11 +82,12 @@ import static org.apache.solr.filestore.TestDistribPackageStore.*;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
public class TestPackages extends SolrCloudTestCase {
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- System.setProperty("enable.packages", "true");
+ System.setProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP, "true");
configureCluster(4)
.withJettyConfig(jetty -> jetty.enableV2(true))
.addConfig("conf", configset("conf2"))
@@ -100,7 +101,7 @@ public class TestPackages extends SolrCloudTestCase {
if (cluster != null) {
cluster.shutdown();
}
- System.clearProperty("enable.packages");
+ System.clearProperty(PackageLoader.ENABLE_PACKAGES_REPO_PROP);
super.tearDown();
}
@@ -114,6 +115,7 @@ public class TestPackages extends SolrCloudTestCase {
}
+
@Test
public void testCoreReloadingPlugin() throws Exception {
String FILE1 = "/mypkg/runtimelibs.jar";
@@ -517,7 +519,7 @@ public class TestPackages extends SolrCloudTestCase {
}
}
- private void verifyComponent(SolrClient client, String COLLECTION_NAME,
+ static void verifyComponent(SolrClient client, String COLLECTION_NAME,
String componentType, String componentName, String pkg, String version) throws Exception {
SolrParams params = new MapSolrParams(Map.of("collection", COLLECTION_NAME,
WT, JAVABIN,
diff --git a/solr/modules/build.gradle b/solr/modules/build.gradle
new file mode 100644
index 0000000..c387f6b
--- /dev/null
+++ b/solr/modules/build.gradle
@@ -0,0 +1,71 @@
+import groovy.json.JsonOutput
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+description = 'Solr Modules'
+
+ext {
+ packagingDir = file("$buildDir/packaging")
+ localPackagesFile = "$packagingDir/local_packages.json"
+}
+
+configurations {
+ modulePackages
+ packaging
+}
+
+dependencies {
+ [":solr:modules:extraction",
+ ":solr:modules:scripting"
+ ].each { moduleName ->
+ modulePackages project(path: moduleName, configuration: "packaging")
+ }
+
+ packaging files(packagingDir) {
+ builtBy 'createLocalPackagesJson'
+ }
+}
+
+task createLocalPackagesJson {
+ dependsOn configurations.modulePackages
+
+ doLast {
+ def packagesJson = [
+ packages: [:]
+ ]
+ configurations.modulePackages.each { depPath ->
+ def moduleName = depPath.parentFile.parentFile.name
+ def module = [
+ version: version,
+ moduleFiles: [],
+ manifest: "",
+ ]
+ fileTree(depPath).filter { it.isFile() }.files.each {pkgFile ->
+ def filePath = depPath.toPath().relativize(pkgFile.toPath()).toString()
+ if (filePath.endsWith("manifest.json")) {
+ module.manifest = filePath
+ } else if (filePath.contains("/lib/")) {
+ module.moduleFiles.add(filePath)
+ }
+ }
+ packagesJson.packages[moduleName] = module
+ }
+ mkdir packagingDir
+ new File(localPackagesFile).text = JsonOutput.prettyPrint(JsonOutput.toJson(packagesJson))
+ }
+}
diff --git a/solr/modules/extraction/manifest.json b/solr/modules/extraction/manifest.json
new file mode 100644
index 0000000..6b82948
--- /dev/null
+++ b/solr/modules/extraction/manifest.json
@@ -0,0 +1,22 @@
+{
+ "version-constraint": "9",
+ "plugins": [
+ {
+ "name": "update-extraction",
+ "setup-command": {
+ "path": "/api/collections/${collection}/config",
+ "payload": {"add-requesthandler": {"name": "${NAME}", "class": "solr-extraction:org.apache.solr.handler.extraction.ExtractingRequestHandler"}},
+ "method": "POST"
+ },
+ "uninstall-command": {
+ "path": "/api/collections/${collection}/config",
+ "payload": {"delete-requesthandler": "${NAME}"},
+ "method": "POST"
+ }
+ }
+ ],
+ "parameter-defaults": {
+ "NAME": "/update/extract"
+ }
+ }
+
diff --git a/solr/modules/scripting/manifest.json b/solr/modules/scripting/manifest.json
new file mode 100644
index 0000000..cd3b28d
--- /dev/null
+++ b/solr/modules/scripting/manifest.json
@@ -0,0 +1,23 @@
+{
+ "version-constraint": "10",
+ "plugins": [
+ {
+ "name": "scripting",
+ "setup-command": {
+ "path": "/api/collections/${collection}/config",
+ "payload": {"add-updateprocessor": {"name": "${NAME}", "class": "solr-scripting:org.apache.solr.scripting.update.ScriptUpdateProcessorFactory", "script": "${SCRIPT}"}},
+ "method": "POST"
+ },
+ "uninstall-command": {
+ "path": "/api/collections/${collection}/config",
+ "payload": {"delete-updateprocessor": "${NAME}"},
+ "method": "POST"
+ }
+ }
+ ],
+ "parameter-defaults": {
+ "NAME": "scripting",
+ "SCRIPT": "update-script.js"
+ }
+ }
+
diff --git a/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java b/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java
index 4543b97..804cca7 100644
--- a/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java
+++ b/solr/modules/scripting/src/java/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.java
@@ -224,6 +224,7 @@ public class ScriptUpdateProcessorFactory extends UpdateRequestProcessorFactory
}
@Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
public void inform(SolrCore core) {
if (!core.getCoreDescriptor().isConfigSetTrusted()) {
throw new SolrException(ErrorCode.UNAUTHORIZED, "The configset for this collection was uploaded without any authentication in place,"
@@ -232,8 +233,12 @@ public class ScriptUpdateProcessorFactory extends UpdateRequestProcessorFactory
}
resourceLoader = core.getResourceLoader();
- // test that our engines & scripts are valid
+ // Hack needed to preload some Lucene classes that otherwise don't load via Nashorn
+ // when this scripting module is loaded as a package.
+ // TODO: Preload all Lucene classes from lucene-core*jar and Solr classes from solr-core*jar
+ core.getResourceLoader().findClass("org.apache.lucene.index.Term", Object.class);
+ // test that our engines & scripts are valid
SolrQueryResponse rsp = new SolrQueryResponse();
SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
try {
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index aa9e533..f3eed5c 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -471,6 +471,7 @@ public class MiniSolrCloudCluster {
JettySolrRunner jetty = !trackJettyMetrics
? new JettySolrRunner(runnerPath.toString(), nodeProps, newConfig)
: new JettySolrRunnerWithMetrics(runnerPath.toString(), nodeProps, newConfig);
+ if(config.preStartupHook != null) config.preStartupHook.accept(jetty);
jetty.start();
jettys.add(jetty);
synchronized (startupWait) {