You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by rw...@apache.org on 2013/12/18 08:11:51 UTC
svn commit: r1551872 - in
/stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr:
SolrServerAdapter.java impl/OsgiResourceLoaderUtil.java
impl/OsgiSolrResourceLoader.java impl/OsgiZkSolrResourceLoader.java
Author: rwesten
Date: Wed Dec 18 07:11:50 2013
New Revision: 1551872
URL: http://svn.apache.org/r1551872
Log:
fix for STANBOL-1235; first steps for STANBOL-1243 (ZooKeeper support)
Added:
stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiResourceLoaderUtil.java
stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiZkSolrResourceLoader.java
Modified:
stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/SolrServerAdapter.java
stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiSolrResourceLoader.java
Modified: stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/SolrServerAdapter.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/SolrServerAdapter.java?rev=1551872&r1=1551871&r2=1551872&view=diff
==============================================================================
--- stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/SolrServerAdapter.java (original)
+++ stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/SolrServerAdapter.java Wed Dec 18 07:11:50 2013
@@ -54,18 +54,26 @@ import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.FilenameUtils;
+import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.ConfigSolr;
+import org.apache.solr.core.ConfigSolrXml;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.core.ZkContainer;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.stanbol.commons.solr.impl.OsgiSolrResourceLoader;
+import org.apache.stanbol.commons.solr.impl.OsgiZkSolrResourceLoader;
+import org.apache.zookeeper.KeeperException;
import org.osgi.framework.BundleContext;
import org.osgi.framework.Constants;
import org.osgi.framework.ServiceReference;
@@ -328,6 +336,7 @@ public class SolrServerAdapter {
public void reloadCore(String name) throws ParserConfigurationException, IOException, SAXException {
//try to reload
log.info("Reload Core {} on CoreContainer {}",name,serverProperties.getServerName());
+
ClassLoader classLoader = updateContextClassLoader();
try {
//TODO: what happens if the core with 'name' is no longer present?
@@ -385,6 +394,11 @@ public class SolrServerAdapter {
new Object[]{core1,core2, serverProperties.getServerName()});
//swap the cores
server.swap(core1, core2);
+ //if succeeded (re-)register the swapped core
+ registerCoreService(core1,null);
+ registerCoreService(core2,null);
+ //update the OSGI service for the CoreContainer
+ updateServerRegistration();
}
/**
@@ -487,16 +501,31 @@ public class SolrServerAdapter {
* {@link SolrCore#getOpenCount()}.
*/
protected ServiceReference registerCoreService(String name,SolrCore core) {
- //first create the new and only than unregister the old (to ensure that
- //the reference count of the SolrCore does not reach 0)
- CoreRegistration current = new CoreRegistration(name,core);
- CoreRegistration old = registrations.put(name,current);
- log.info("added Registration for SolrCore {}",name);
- if(old != null){
- log.info(" ... unregister old registration {}", old);
- old.unregister();
+ //STANBOL-1235: we want to unregister the old before registering the new
+ // but we do not want all solrCores to be closed as otherwise the
+ // SolrCore would be deactivated/activated. So if we find a old
+ // registration we will acquire a 2nd reference to the same core
+ // for the time of the re-registration
+ SolrCore sameCore = null;
+ try {
+ CoreRegistration current;
+ synchronized (registrations) {
+ CoreRegistration old = registrations.remove(name);
+ if(old != null){
+ sameCore = this.server.getCore(name); //2nd reference to the core
+ log.info(" ... unregister old registration {}", old);
+ old.unregister();
+ }
+ current = new CoreRegistration(name,core);
+ registrations.put(name,current);
+ }
+ log.info(" ... register {}",current);
+ return current.getServiceReference();
+ } finally {
+ if(sameCore != null){ //clean up the 2nd reference
+ sameCore.close();
+ }
}
- return current.getServiceReference();
}
/**
@@ -628,58 +657,107 @@ public class SolrServerAdapter {
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Solr has shutdown.");
}
log.info(" .... createCore {}:{}",serverProperties.getServerName(),dcore.getName());
+ File idir = new File(dcore.getInstanceDir());
+ String instanceDir = idir.getPath();
+ SolrCore created;
if (getZkController() != null) {
- //TODO: add support for ZooKeeper managed cores
- return super.create(dcore);
+ created = createFromZk(instanceDir, dcore);
} else {
- File idir = new File(dcore.getInstanceDir());
- String instanceDir = idir.getPath();
- SolrResourceLoader loader = new OsgiSolrResourceLoader(context, instanceDir,
- CoreContainer.class.getClassLoader());
- SolrConfig config;
- try {
- config = new SolrConfig(loader, dcore.getConfigName(), null);
- } catch (Exception e) {
- log.error("Failed to load file {}", new File(instanceDir, dcore.getConfigName()).getAbsolutePath());
- throw new SolrException(ErrorCode.SERVER_ERROR, "Could not load config for " + dcore.getConfigName(), e);
- }
- IndexSchema schema = null;
- //indexSchemaCache is now protected (Solr 4.4)
- if (indexSchemaCache != null) {
- final String resourceNameToBeUsed = IndexSchemaFactory.getResourceNameToBeUsed(dcore.getSchemaName(), config);
- File schemaFile = new File(resourceNameToBeUsed);
- if (!schemaFile.isAbsolute()) {
- schemaFile = new File(loader.getConfigDir(), schemaFile.getPath());
- }
- if (schemaFile.exists()) {
- String key = schemaFile.getAbsolutePath()
- + ":"
- + new SimpleDateFormat("yyyyMMddHHmmss", Locale.ROOT).format(new Date(
- schemaFile.lastModified()));
- schema = indexSchemaCache.get(key);
- if (schema == null) {
- log.info("creating new schema object for core: " + dcore.getProperty(CoreDescriptor.CORE_NAME));
- schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(), config);
- indexSchemaCache.put(key, schema);
- } else {
- log.info("re-using schema object for core: " + dcore.getProperty(CoreDescriptor.CORE_NAME));
- }
- }
- }
+ created = createFromLocal(dcore, instanceDir);
+ }
+ //TODO: solrCores is private ...
+ //solrCores.addCreated(created); // For persisting newly-created cores.
+ return created;
+ }
+ /*
+ * Create from local configuration (replaces the method with the same
+ * name in the parent class)
+ */
+ private SolrCore createFromLocal(CoreDescriptor dcore, String instanceDir) {
+ SolrResourceLoader loader = new OsgiSolrResourceLoader(context, instanceDir,
+ CoreContainer.class.getClassLoader());
+ SolrConfig config;
+ try {
+ config = new SolrConfig(loader, dcore.getConfigName(), null);
+ } catch (Exception e) {
+ log.error("Failed to load file {}", new File(instanceDir, dcore.getConfigName()).getAbsolutePath());
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not load config for " + dcore.getConfigName(), e);
+ }
+ IndexSchema schema = null;
+ if (indexSchemaCache != null) {
+ final String resourceNameToBeUsed = IndexSchemaFactory.getResourceNameToBeUsed(dcore.getSchemaName(), config);
+ File schemaFile = new File(resourceNameToBeUsed);
+ if (!schemaFile.isAbsolute()) {
+ schemaFile = new File(loader.getConfigDir(), schemaFile.getPath());
+ }
+ if (schemaFile.exists()) {
+ String key = schemaFile.getAbsolutePath()
+ + ":"
+ + new SimpleDateFormat("yyyyMMddHHmmss", Locale.ROOT).format(new Date(
+ schemaFile.lastModified()));
+ schema = indexSchemaCache.get(key);
if (schema == null) {
+ log.info("creating new schema object for core: " + dcore.getProperty(CoreDescriptor.CORE_NAME));
schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(), config);
+ indexSchemaCache.put(key, schema);
+ } else {
+ log.info("re-using schema object for core: " + dcore.getProperty(CoreDescriptor.CORE_NAME));
}
+ }
+ }
- SolrCore core = new SolrCore(dcore.getName(), null, config, schema, dcore);
- if (core.getUpdateHandler().getUpdateLog() != null) {
- // always kick off recovery if we are in standalone mode.
- core.getUpdateHandler().getUpdateLog().recoverFromLog();
- }
- return core;
+ if (schema == null) {
+ schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(), config);
}
+
+ SolrCore core = new SolrCore(dcore.getName(), null, config, schema, dcore);
+ if (core.getUpdateHandler().getUpdateLog() != null) {
+ // always kick off recovery if we are in standalone mode.
+ core.getUpdateHandler().getUpdateLog().recoverFromLog();
+ }
+ return core;
}
+ /*
+ * Create from Zookeeper (replaces the method with the same name in
+ * {@link ZkContainer})
+ */
+ private SolrCore createFromZk(String instanceDir, CoreDescriptor dcore) {
+ try {
+ SolrResourceLoader solrLoader = null;
+ SolrConfig config = null;
+ String zkConfigName = null;
+ IndexSchema schema;
+ String collection = dcore.getCloudDescriptor().getCollectionName();
+ ZkController zkController = getZkController();
+ zkController.createCollectionZkNode(dcore.getCloudDescriptor());
+
+ zkConfigName = zkController.readConfigName(collection);
+ if (zkConfigName == null) {
+ log.error("Could not find config name for collection:" + collection);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "Could not find config name for collection:" + collection);
+ }
+ solrLoader = new OsgiZkSolrResourceLoader(context, instanceDir, zkConfigName,
+ CoreContainer.class.getClassLoader(),
+ // TODO: Core properties are not accessible -> parse null
+ // ConfigSolrXml.getCoreProperties(instanceDir, dcore),
+ null, zkController);
+ config = zkSys.getSolrConfigFromZk(zkConfigName, dcore.getConfigName(), solrLoader);
+ schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(), config);
+ return new SolrCore(dcore.getName(), null, config, schema, dcore);
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ }
+ }
+
//this ensures that a closeHook is added to registered cores
@Override
protected SolrCore registerCore(boolean isTransientCore, String name, SolrCore core, boolean returnPrevNotClosed) {
Added: stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiResourceLoaderUtil.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiResourceLoaderUtil.java?rev=1551872&view=auto
==============================================================================
--- stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiResourceLoaderUtil.java (added)
+++ stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiResourceLoaderUtil.java Wed Dec 18 07:11:50 2013
@@ -0,0 +1,80 @@
+package org.apache.stanbol.commons.solr.impl;
+
+import static org.apache.stanbol.commons.solr.SolrConstants.PROPERTY_ANALYZER_FACTORY_NAME;
+
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.stanbol.commons.solr.SolrConstants;
+import org.apache.stanbol.commons.solr.utils.RegisteredSolrAnalyzerFactory;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.InvalidSyntaxException;
+import org.osgi.framework.ServiceReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OsgiResourceLoaderUtil {
+
+ /*
+ * static members form the SolrResourceLoader that are not visible to this
+ * class
+ */
+ static final String project = "solr";
+ static final String base = "org.apache" + "." + project;
+ private static final Pattern legacyAnalysisPattern = Pattern.compile("((\\Q" + base
+ + ".analysis.\\E)|(\\Q" + project
+ + ".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
+
+ private static Logger log = LoggerFactory.getLogger(OsgiResourceLoaderUtil.class);
+
+ /**
+ * Finds classes by using {@link RegisteredSolrAnalyzerFactory} with a
+ * filter over {@link SolrConstants#PROPERTY_ANALYZER_FACTORY_NAME}.
+ * @param bc the {@link BundleContext} used for the search
+ * @param cname the cname as parsed to {@link SolrResourceLoader#findClass(String, Class, String...)}
+ * @param expectedType the expected type as parsed to {@link SolrResourceLoader#findClass(String, Class, String...)}
+ * @param subpackages the subpackages as parsed to {@link SolrResourceLoader#findClass(String, Class, String...)}
+ * @return the class
+ */
+ public static <T> Class<? extends T> findOsgiClass(BundleContext bc, String cname, Class<T> expectedType, String... subpackages){
+ Class<? extends T> clazz = null;
+ RuntimeException parentEx = null;
+ final Matcher m = legacyAnalysisPattern.matcher(cname);
+ if (m.matches()) {
+ final String name = m.group(4);
+ log.trace("Trying to load class from analysis SPI using name='{}'", name);
+ ServiceReference[] referenced;
+ String filter;
+ try {
+ filter = String.format("(%s=%s)", PROPERTY_ANALYZER_FACTORY_NAME, name.toLowerCase(Locale.ROOT));
+ referenced = bc.getServiceReferences(RegisteredSolrAnalyzerFactory.class.getName(), filter);
+ } catch (InvalidSyntaxException e) {
+ throw new IllegalStateException("Unable to create Filter for Service with name '" + name
+ + "'!", e);
+ }
+ if (referenced != null && referenced.length > 0) {
+ Object service = bc.getService(referenced[0]);
+ if (service instanceof RegisteredSolrAnalyzerFactory) {
+ //TODO: we could check the type here
+ clazz = ((RegisteredSolrAnalyzerFactory)service).getFactoryClass();
+ //we do not use a service so immediately unget it
+ bc.ungetService(referenced[0]);
+ return clazz;
+ }
+ } else {
+ parentEx = new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Error loading Class '" + cname + "' via OSGI service Registry by using filter '"
+ + filter + "'!", parentEx);
+ }
+ }
+ if(parentEx != null) {
+ throw parentEx;
+ } else {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error loading class '" + cname + "'");
+ }
+ }
+
+}
Modified: stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiSolrResourceLoader.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiSolrResourceLoader.java?rev=1551872&r1=1551871&r2=1551872&view=diff
==============================================================================
--- stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiSolrResourceLoader.java (original)
+++ stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiSolrResourceLoader.java Wed Dec 18 07:11:50 2013
@@ -90,40 +90,19 @@ public class OsgiSolrResourceLoader exte
}
if (clazz != null) {
return clazz;
- }
- final Matcher m = legacyAnalysisPattern.matcher(cname);
- if (m.matches()) {
- final String name = m.group(4);
- log.trace("Trying to load class from analysis SPI using name='{}'", name);
- ServiceReference[] referenced;
- String filter;
+ } else {
try {
- filter = String.format("(%s=%s)", PROPERTY_ANALYZER_FACTORY_NAME, name.toLowerCase(Locale.ROOT));
- referenced = bc.getServiceReferences(RegisteredSolrAnalyzerFactory.class.getName(), filter);
- } catch (InvalidSyntaxException e) {
- throw new IllegalStateException("Unable to create Filter for Service with name '" + name
- + "'!", e);
- }
- if (referenced != null && referenced.length > 0) {
- Object service = bc.getService(referenced[0]);
- if (service instanceof RegisteredSolrAnalyzerFactory) {
- //TODO: we could check the type here
- clazz = ((RegisteredSolrAnalyzerFactory)service).getFactoryClass();
- //we do not use a service so immediately unget it
- bc.ungetService(referenced[0]);
- return clazz;
+ //try to load via the OSGI service factory
+ return OsgiResourceLoaderUtil.findOsgiClass(bc, cname, expectedType, subpackages);
+ } catch (SolrException e) {
+ //prefer to throw the first exception
+ if(parentEx != null){
+ throw parentEx;
+ } else {
+ throw e;
}
- } else {
- parentEx = new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error loading Class '" + cname + "' via OSGI service Registry by using filter '"
- + filter + "'!", parentEx);
}
}
- if(parentEx != null) {
- throw parentEx;
- } else {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error loading class '" + cname + "'");
- }
}
}
Added: stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiZkSolrResourceLoader.java
URL: http://svn.apache.org/viewvc/stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiZkSolrResourceLoader.java?rev=1551872&view=auto
==============================================================================
--- stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiZkSolrResourceLoader.java (added)
+++ stanbol/trunk/commons/solr/core/src/main/java/org/apache/stanbol/commons/solr/impl/OsgiZkSolrResourceLoader.java Wed Dec 18 07:11:50 2013
@@ -0,0 +1,60 @@
+package org.apache.stanbol.commons.solr.impl;
+
+import java.util.Properties;
+
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.osgi.framework.BundleContext;
+
+/**
+ * Extends the {@link ZkSolrResourceLoader} to support findClass(..) methods
+ * to load classes via OSGI.
+ *
+ * @author Rupert Westenthaler
+ *
+ */
+public class OsgiZkSolrResourceLoader extends ZkSolrResourceLoader {
+
+
+ protected final BundleContext bc;
+
+ public OsgiZkSolrResourceLoader(BundleContext bc, String instanceDir,
+ String collection, ZkController zooKeeperController) {
+ super(instanceDir,collection,zooKeeperController);
+ this.bc = bc;
+ }
+
+ public OsgiZkSolrResourceLoader(BundleContext bc, String instanceDir,
+ String collection, ClassLoader parent, Properties coreProperties,
+ ZkController zooKeeperController) {
+ super(instanceDir, collection, parent, coreProperties, zooKeeperController);
+ this.bc = bc;
+ }
+
+ @Override
+ public <T> Class<? extends T> findClass(String cname, Class<T> expectedType, String... subpackages) {
+ Class<? extends T> clazz = null;
+ RuntimeException parentEx = null;
+ try {
+ clazz = super.findClass(cname, expectedType, subpackages);
+ } catch (RuntimeException e) {
+ parentEx = e;
+ }
+ if (clazz != null) {
+ return clazz;
+ } else {
+ try {
+ //try to load via the OSGI service factory
+ return OsgiResourceLoaderUtil.findOsgiClass(bc, cname, expectedType, subpackages);
+ } catch (SolrException e) {
+ //prefer to throw the first exception
+ if(parentEx != null){
+ throw parentEx;
+ } else {
+ throw e;
+ }
+ }
+ }
+ }
+}