You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by su...@apache.org on 2012/09/20 13:22:17 UTC
svn commit: r1387977 [2/3] - in
/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub:
defaults/src/main/resources/config/
index/src/main/java/org/apache/stanbol/contenthub/index/
index/src/main/java/org/apache/stanbol/contenthub/inde...
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndex.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndex.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndex.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndex.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.index.solr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Dictionary;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.clerezza.rdf.core.Triple;
+import org.apache.clerezza.rdf.core.UriRef;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.ConfigurationPolicy;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.LukeRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.stanbol.commons.semanticindex.index.EndpointTypeEnum;
+import org.apache.stanbol.commons.semanticindex.index.IndexException;
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.commons.semanticindex.index.IndexState;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndex;
+import org.apache.stanbol.commons.semanticindex.store.ChangeSet;
+import org.apache.stanbol.commons.semanticindex.store.IndexingSource;
+import org.apache.stanbol.commons.semanticindex.store.Store;
+import org.apache.stanbol.commons.semanticindex.store.StoreException;
+import org.apache.stanbol.commons.solr.IndexReference;
+import org.apache.stanbol.commons.solr.RegisteredSolrServerTracker;
+import org.apache.stanbol.commons.solr.managed.IndexMetadata;
+import org.apache.stanbol.commons.solr.managed.ManagedSolrServer;
+import org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex;
+import org.apache.stanbol.contenthub.servicesapi.index.search.featured.FeaturedSearch;
+import org.apache.stanbol.contenthub.servicesapi.index.search.solr.SolrSearch;
+import org.apache.stanbol.contenthub.servicesapi.store.vocabulary.SolrVocabulary.SolrFieldName;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.entityhub.core.utils.OsgiUtils;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.Constants;
+import org.osgi.framework.InvalidSyntaxException;
+import org.osgi.framework.ServiceReference;
+import org.osgi.service.cm.Configuration;
+import org.osgi.service.cm.ConfigurationAdmin;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * LDPath based {@link SemanticIndex} implementation. This implementations creates the underlying Solr core by
+ * parsing the provided LDPath program. Several LDPath based semantic indexes can be created through the
+ * associated RESTful services deployed under {stanbolhost}/contenthub/index/ldpath or through the Felix Web
+ * Console.
+ *
+ * Following parameters can be configured while creating a SemanticIndex:
+ * <ul>
+ * <li><b>Name:</b> Name of the index</li>
+ * <li><b>Description:</b> Description of the index</li>
+ * <li><b>LDPathProgram: </b> LDPath program that will be used as a source to create the semantic index. Index
+ * fields and Solr specific configurations regarding those index fields are given in this parameter.</li>
+ * <li><b>Index Content: </b> If this configuration is true plain text content of the {@link ContentItem} is
+ * also indexed to be used in the full text search.</li>
+ * <li><b>Batch Size:</b> Maximum number of changes to be processed in a single step while iteratively
+ * checking the changes in the {@link IndexingSource}</li>
+ * <li><b>Indexing Source Name: </b> Name of the {@link IndexingSource} instance to be checked for updates.
+ * This {@link IndexingSource} should be an IndexingSource<ContentItem> implementation.
+ * <li><b>Indexing Source Check Period:</b> Time to check changes in the {@link IndexingSource} in second
+ * units</li>
+ * <li><b>Solr Server Check Time:</b> Maximum time in seconds to wait for the availability of the Solr core
+ * associated with this index</li>
+ * <li><b>Service Ranking:</b> To be able adjust priorities of {@link SemanticIndex}es with same name or same
+ * {@link EndpointTypeEnum}, this property is used. The higher value of this property, the higher priority of
+ * the {@link SemanticIndex} instance.</li>
+ * </ul>
+ *
+ * @author suat
+ * @author meric
+ *
+ */
+@Component(configurationFactory = true, policy = ConfigurationPolicy.REQUIRE, metatype = true, immediate = true)
+@Service
+@Properties(value = {@Property(name = SolrSemanticIndex.PROP_SOLR_CHECK_TIME, intValue = 5),
+ @Property(name = SolrSemanticIndex.PROP_INDEX_CONTENT, boolValue = true)})
+public class SolrSemanticIndex extends AbstractLDPathSemanticIndex {
+
+ public static final String PROP_SOLR_CHECK_TIME = "org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.solrCheckTime";
+ public static final String PROP_INDEX_CONTENT = "org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.indexContent";
+
+ private static final Logger logger = LoggerFactory.getLogger(SolrSemanticIndex.class);
+
+ private static final Set<String> SUPPORTED_MIMETYPES = Collections.unmodifiableSet(new HashSet<String>(
+ Arrays.asList(MediaType.TEXT_HTML, MediaType.TEXT_PLAIN, MediaType.TEXT_XML)));
+
+ private boolean indexContent;
+
+ @Reference
+ private SolrSemanticIndexFactory solrSemanticIndexFactory;
+
+ @Reference(target = "(org.apache.solr.core.CoreContainer.name=contenthub)")
+ private ManagedSolrServer managedSolrServer;
+
+ private int solrCheckTime;
+ private RegisteredSolrServerTracker registeredServerTracker;
+
+ @Activate
+ protected void activate(ComponentContext context) throws IndexException,
+ IndexManagementException,
+ ConfigurationException,
+ StoreException {
+
+ super.activate(context);
+ @SuppressWarnings("rawtypes")
+ Dictionary properties = context.getProperties();
+ this.solrCheckTime = (Integer) OsgiUtils.checkProperty(properties, PROP_SOLR_CHECK_TIME);
+ this.indexContent = (Boolean) OsgiUtils.checkProperty(properties,
+ SolrSemanticIndex.PROP_INDEX_CONTENT);
+
+ // create index if it is not already done. When an instance of this component is created through the
+ // REST/Java services first the Solr core initalized and then the associated OSGI component is
+ // activated
+ if (!solrSemanticIndexFactory.getSemanticIndexMetadataManager().isConfigured(pid)) {
+ // solr core has not been created. create now...
+ logger.info("New Solr core will be created for the Semantic Index: {}", this.name);
+ solrSemanticIndexFactory.createIndex(getConfigProperties());
+ this.state = IndexState.ACTIVE;
+ this.epoch = indexingSource.getEpoch();
+
+ } else {
+ // solr core has already been created or the semantic index component is re-activated second or
+ // more
+
+ // check the configuration has changed
+ java.util.Properties oldMetadata = solrSemanticIndexFactory.getSemanticIndexMetadataManager()
+ .getIndexMetadata(pid);
+
+ checkUnmodifiableConfigurations(name, indexingSource.getName(), oldMetadata);
+ if (checkReindexingCondition(ldPathProgram, oldMetadata)) {
+ logger.info(
+ "LDPath program of the Semantic Index: {} has been changed. Reindexing will start now...",
+ this.name);
+ // ldpath has changed, reindexing is needed
+ this.state = IndexState.REINDEXING;
+ this.epoch = Long.parseLong((String) oldMetadata.getProperty(SemanticIndex.PROP_EPOCH));
+
+ startReindexing();
+
+ } else {
+ if (oldMetadata.get(SemanticIndex.PROP_REVISION) != null) {
+ // load revision of the index and update the index state
+ this.revision = Long.parseLong((String) oldMetadata.get(SemanticIndex.PROP_REVISION));
+ this.epoch = Long.parseLong((String) oldMetadata.getProperty(SemanticIndex.PROP_EPOCH));
+ this.state = IndexState.valueOf(oldMetadata.getProperty(SemanticIndex.PROP_STATE));
+
+ } else {
+ // newly created index, store the metadata. Index was created through the REST/Java
+ // services
+ this.state = IndexState.ACTIVE;
+ this.epoch = indexingSource.getEpoch();
+ }
+ }
+ }
+
+ if (this.state != IndexState.REINDEXING) {
+ // start tracking for the Solr core
+ initializeTracker(this.name);
+
+ // start polling the changes in the store
+ startIndexingSourceCheckThread();
+ }
+ updateIndexMetadata();
+ logger.info("The SemanticIndex: {} initialized successfully", this.name);
+ }
+
+ private void initializeTracker(String name) {
+ try {
+ registeredServerTracker = new RegisteredSolrServerTracker(componentContext.getBundleContext(),
+ new IndexReference(managedSolrServer.getServerName(), name), null);
+ } catch (InvalidSyntaxException e) {
+ // ignore as there is no filter specified
+ }
+
+ // start tracking
+ registeredServerTracker.open();
+ }
+
+ @Deactivate
+ protected void deactivate(ComponentContext context) {
+ super.deactivate(context);
+ if (registeredServerTracker != null) {
+ registeredServerTracker.close();
+ }
+
+ // check the configuration is deleted or just deactivated
+ ServiceReference reference = context.getBundleContext().getServiceReference(
+ ConfigurationAdmin.class.getName());
+ ConfigurationAdmin configAdmin = (ConfigurationAdmin) context.getBundleContext()
+ .getService(reference);
+ Configuration config;
+ try {
+ config = configAdmin.getConfiguration(this.pid);
+ // if the configuration for the index has been removed, clear all files for this index
+ if (config.getProperties() == null) {
+ logger.info(
+ "Configuration for the Semantic Index: {} has been deleted. All resources will be removed.",
+ this.name);
+ if (solrSemanticIndexFactory.getSemanticIndexMetadataManager().isConfigured(pid)) {
+ // this case is a check for the remove operation from the felix web console
+ solrSemanticIndexFactory.removeIndex(this.pid);
+ }
+ } // the index is deactivated. do not nothing.
+ } catch (IOException e) {
+ logger.error("Failed to obtain configuration for the Semantic Index: {}.", this.name, e);
+ } catch (IndexManagementException e) {
+ logger.error("Failed to remove index: {} while deactivating", this.name, e);
+ }
+ }
+
+ /**
+ * This implementation of {@link #index(ContentItem)} method first, gets the enhancements having
+ * {@link org.apache.stanbol.enhancer.servicesapi.rdf.Properties#ENHANCER_ENTITY_REFERENCE} property.
+ * Target values (a set of entities declared in different external sites) are queried from the Entityhub.
+ * During the querying operation the LDPath program which was used to create this index is used. Obtained
+ * results are indexed along with the actual content.
+ */
+ @Override
+ public boolean index(ContentItem ci) throws IndexException {
+ if (this.state == IndexState.REINDEXING) {
+ throw new IndexException(String.format(
+ "The index '%s' is read-only as it is in reindexing state.", name));
+ }
+ semUp();
+ try {
+ performIndex(ci);
+ } finally {
+ semDown();
+ }
+ return true;
+ }
+
+ private void performIndex(ContentItem ci) throws IndexException {
+ try {
+ SolrServer solrServer = getServer();
+ SolrInputDocument doc = getSolrDocument(ci);
+ if (this.indexContent && SUPPORTED_MIMETYPES.contains(ci.getMimeType())) {
+ doc.addField(SolrFieldName.CONTENT.toString(),
+ org.apache.commons.io.IOUtils.toString(ci.getStream()));
+ }
+ solrServer.add(doc);
+ solrServer.commit();
+ logger.debug("Documents are committed to Solr Server successfully.");
+
+ } catch (SolrServerException e) {
+ logger.error("Given SolrInputDocument cannot be added to Solr Server with name " + this.name, e);
+ throw new IndexException("Given SolrInputDocument cannot be added to Solr Server with name "
+ + this.name, e);
+ } catch (IOException e) {
+ logger.error("Given SolrInputDocument cannot be added to Solr Server with name " + this.name, e);
+ throw new IndexException("Given SolrInputDocument cannot be added to Solr Server with name "
+ + this.name, e);
+ } catch (IndexManagementException e) {
+ logger.error("Cannot execute the ldPathProgram on ContentItem's metadata", e);
+ throw new IndexException("Cannot execute the ldPathProgram on ContentItem's metadata", e);
+ }
+ }
+
+ private SolrInputDocument getSolrDocument(ContentItem ci) throws IndexManagementException {
+ Iterator<Triple> it = ci.getMetadata().filter(null,
+ org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_ENTITY_REFERENCE, null);
+ Set<String> contexts = new HashSet<String>();
+ while (it.hasNext()) {
+ Resource r = it.next().getObject();
+ if (r instanceof UriRef) {
+ contexts.add(((UriRef) r).getUnicodeString());
+ }
+ }
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField(SolrFieldName.ID.toString(), ci.getUri().getUnicodeString());
+ Map<String,Collection<?>> results = executeProgram(contexts, ci);
+ for (Entry<String,Collection<?>> entry : results.entrySet()) {
+ doc.addField(entry.getKey(), entry.getValue());
+ }
+
+ return doc;
+ }
+
+ @Override
+ public void remove(String uri) throws IndexException {
+ if (this.state == IndexState.REINDEXING) {
+ throw new IndexException(String.format(
+ "The index '%s' is read-only as it is in reindexing state.", name));
+ }
+
+ semUp();
+ try {
+ performRemove(uri);
+ } finally {
+ semDown();
+ }
+ }
+
+ private void performRemove(String ciURI) throws IndexException {
+ if (ciURI == null || ciURI.isEmpty()) {
+ return;
+ }
+
+ SolrServer solrServer = null;
+ try {
+ solrServer = getServer();
+ solrServer.deleteById(ciURI);
+ solrServer.commit();
+ logger.info("Given Uri {} is removed from index successfully", ciURI);
+ } catch (SolrServerException e) {
+ logger.error("Given SolrInputDocument cannot be added to Solr Server with name " + this.name, e);
+ throw new IndexException("Given SolrInputDocument cannot be added to Solr Server with name "
+ + this.name, e);
+ } catch (IOException e) {
+ logger.error("Given SolrInputDocument cannot be added to Solr Server with name " + this.name, e);
+ throw new IndexException("Given SolrInputDocument cannot be added to Solr Server with name "
+ + this.name, e);
+ }
+ }
+
+ @Override
+ public List<String> getFieldsNames() throws IndexException {
+ SolrServer solrServer = null;
+ LukeRequest qr = new LukeRequest();
+ qr.setShowSchema(true);
+ NamedList<Object> qresp = null;
+ try {
+ solrServer = getServer();
+ qresp = solrServer.request(qr);
+ } catch (SolrServerException e) {
+ logger.error("Cannot retrieve fields from solr with Luke Request", e);
+ throw new IndexException("Cannot retrieve field names from solr with Luke Request", e);
+ } catch (IOException e) {
+ logger.error("Cannot retrieve fields from solr with Luke Request", e);
+ throw new IndexException("Cannot retrieve field names from solr with Luke Request", e);
+ }
+ List<String> fieldNames = null;
+ Object schema = qresp.get("schema");
+ @SuppressWarnings("unchecked")
+ NamedList<Object> schemaList = (NamedList<Object>) schema;
+ Object fields = schemaList.get("fields");
+ if (fields instanceof NamedList<?>) {
+ @SuppressWarnings("unchecked")
+ NamedList<Object> fieldsList = (NamedList<Object>) fields;
+ fieldNames = new ArrayList<String>();
+ for (int i = 0; i < fieldsList.size(); i++) {
+ fieldNames.add(fieldsList.getName(i));
+ }
+ }
+ return (fieldNames == null || fieldNames.size() == 0) ? null : fieldNames;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map<String,Object> getFieldProperties(String name) throws IndexException {
+ SolrServer solrServer = null;
+ Map<String,Object> fieldProperties = new HashMap<String,Object>();
+
+ LukeRequest qr = new LukeRequest();
+ qr.setShowSchema(true);
+ NamedList<Object> qresp = null;
+ try {
+ solrServer = getServer();
+ qresp = solrServer.request(qr);
+ } catch (SolrServerException e) {
+ logger.error("Cannot retrieve fields from solr with Luke Request", e);
+ throw new IndexException("Cannot retrieve field names from solr with Luke Request", e);
+ } catch (IOException e) {
+ logger.error("Cannot retrieve fields from solr with Luke Request", e);
+ throw new IndexException("Cannot retrieve field names from solr with Luke Request", e);
+ }
+ Object schema = qresp.get("schema");
+ NamedList<Object> schemaList = (NamedList<Object>) schema;
+ Object fields = schemaList.get("fields");
+ if (fields instanceof NamedList<?>) {
+
+ NamedList<Object> fieldsList = (NamedList<Object>) fields;
+ Object field = fieldsList.get(name);
+ if (field instanceof NamedList<?>) {
+ NamedList<Object> fieldProps = (NamedList<Object>) field;
+ for (int i = 0; i < fieldProps.size(); i++) {
+ fieldProperties.put(fieldProps.getName(i), fieldProps.getVal(i));
+ }
+ }
+ } else {
+ throw new IllegalStateException(
+ "Fields container is not a NamedList, so there is no facet information available");
+ }
+ return (fieldProperties == null || fieldProperties.size() == 0) ? null : fieldProperties;
+ }
+
+ @Override
+ public Map<String,String> getRESTSearchEndpoints() {
+ Map<String,String> searchEndpoints = new HashMap<String,String>();
+ searchEndpoints.put(EndpointTypeEnum.CONTENTHUB.getUri(), "contenthub/" + this.name
+ + "/search/featured");
+ searchEndpoints.put(EndpointTypeEnum.SOLR.getUri(), "solr/" + managedSolrServer.getServerName() + "/"
+ + this.name);
+ return searchEndpoints;
+ }
+
+ @Override
+ public Map<String,ServiceReference> getSearchEndPoints() {
+ BundleContext bundleContext = this.componentContext.getBundleContext();
+ Map<String,ServiceReference> serviceEndPoints = new HashMap<String,ServiceReference>();
+ ServiceReference serviceReference = null;
+
+ serviceReference = bundleContext.getServiceReference(SolrSearch.class.getName());
+ if (serviceReference != null) {
+ serviceEndPoints.put(SolrSearch.class.getName(), serviceReference);
+ }
+ serviceReference = bundleContext.getServiceReference(FeaturedSearch.class.getName());
+ if (serviceReference != null) {
+ serviceEndPoints.put(FeaturedSearch.class.getName(), serviceReference);
+ }
+
+ return serviceEndPoints;
+ }
+
+ /**
+ * Obtains the {@link SolrServer} for the given Solr core identified by the name of this Semantic Index.
+ * It uses the {@link RegisteredSolrServerTracker} and waits for the server by {@link #solrCheckTime}.
+ *
+ * @return the {@link SolrServer} instance for the given {@code coreName}.
+ * @throws IndexException
+ */
+ public SolrServer getServer() throws IndexException {
+ SolrServer solrServer = registeredServerTracker.getService();
+
+ // if server is null wait as long as the specified the waiting time for the solr server
+ if (solrServer == null) {
+ for (int i = 0; i < solrCheckTime; i++) {
+ try {
+ logger.info(" ... waiting 1sec for SolrServer");
+ solrServer = (SolrServer) registeredServerTracker.waitForService(1000);
+ } catch (InterruptedException e) {
+
+ }
+ }
+ }
+
+ if (solrServer == null) {
+ logger.error("Failed to obtain SolrServer through RegisteredSolrServerTracker");
+ throw new IndexException(
+ String.format("Failed to obtain SolrServer through RegisteredSolrServerTracker"));
+ }
+ return solrServer;
+ }
+
+ protected void updateIndexMetadata() throws IndexException {
+ java.util.Properties properties = getConfigProperties();
+ try {
+ solrSemanticIndexFactory.getSemanticIndexMetadataManager().updateIndexMetadata(this.pid,
+ properties);
+ } catch (IndexManagementException e) {
+ logger.error("Failed to update the metadata of the index: {}", this.name, e);
+ throw new IndexException(String.format("Failed to update the metadata of the index: %s",
+ this.name), e);
+ }
+ }
+
+ private java.util.Properties getConfigProperties() {
+ @SuppressWarnings("rawtypes")
+ Dictionary properties = componentContext.getProperties();
+ java.util.Properties propertiesSubset = new java.util.Properties();
+ propertiesSubset.put(PROP_NAME, properties.get(PROP_NAME));
+ propertiesSubset.put(PROP_DESCRIPTION, properties.get(PROP_DESCRIPTION));
+ propertiesSubset.put(AbstractLDPathSemanticIndex.PROP_LD_PATH_PROGRAM,
+ properties.get(AbstractLDPathSemanticIndex.PROP_LD_PATH_PROGRAM));
+ propertiesSubset.put(SolrSemanticIndex.PROP_INDEX_CONTENT,
+ properties.get(SolrSemanticIndex.PROP_INDEX_CONTENT));
+ propertiesSubset.put(AbstractLDPathSemanticIndex.PROP_BATCH_SIZE,
+ properties.get(AbstractLDPathSemanticIndex.PROP_BATCH_SIZE));
+ propertiesSubset.put(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME,
+ properties.get(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME));
+ propertiesSubset.put(PROP_SOLR_CHECK_TIME, properties.get(PROP_SOLR_CHECK_TIME));
+ propertiesSubset.put(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_CHECK_PERIOD,
+ properties.get(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_CHECK_PERIOD));
+ propertiesSubset.put(Constants.SERVICE_PID, properties.get(Constants.SERVICE_PID));
+ propertiesSubset.put(Constants.SERVICE_RANKING, properties.get(Constants.SERVICE_RANKING));
+ propertiesSubset.put(PROP_REVISION, this.revision);
+ propertiesSubset.put(PROP_EPOCH, this.epoch);
+ propertiesSubset.put(PROP_STATE, this.state.name());
+ return propertiesSubset;
+ }
+
+ protected void startReindexing() {
+ reindexerThread = new Thread(new Reindexer());
+ reindexerThread.start();
+ }
+
+ /**
+ * Separate thread to perform reindexing operation in the background. It creates a temporary Solr core,
+ * indexes all documents obtained from the {@link Store} using the new LDPath program. After the indexing
+ * operation finishes, the temporary core is replaced with the existing one and the temporary core is
+ * deleted.
+ *
+ * @author suat
+ *
+ */
+ private class Reindexer implements Runnable {
+ @Override
+ public void run() {
+ // create temporary core
+ IndexMetadata temporaryCoreMetadata = null;
+ try {
+ temporaryCoreMetadata = createTemporarySolrCore();
+ logger.info(
+ "Temporary solr core: {} has been created for reindexing of the Semantic Index: {}",
+ temporaryCoreMetadata.getIndexName(), name);
+ } catch (IndexManagementException e) {
+ logger.error("Failed to create temporary Solr core while reindexing the index: {}", name, e);
+ return;
+ }
+ String temporaryCoreName = temporaryCoreMetadata.getIndexName();
+
+ try {
+ // initialize solr server tracker for the temporary core
+ initializeTracker(temporaryCoreName);
+
+ // index documents in the store according to the new configuration
+
+ revision = indexDocuments();
+ logger.info(
+ "Documents have been re-indexed according to the new configuration of the Semantic Index: {}",
+ name);
+ } catch (StoreException e) {
+ logger.error("Failed to obtain changes from Store while reindexing the index: {}", name, e);
+ managedSolrServer.removeIndex(temporaryCoreName, true);
+ return;
+ } catch (IndexException e) {
+ logger.error("IndexException while reindexing the index: {}", name, e);
+ managedSolrServer.removeIndex(temporaryCoreName, true);
+ return;
+ } catch (Exception e) {
+ logger.error("Exception while reindexing the index: {}", name, e);
+ managedSolrServer.removeIndex(temporaryCoreName, true);
+ return;
+ }
+
+ // swap indexes
+ managedSolrServer.swapIndexes(name, temporaryCoreName);
+
+ // remove the old one
+ managedSolrServer.removeIndex(temporaryCoreName, true);
+
+ // adjust the tracker so that it tracks the actual solr core
+ registeredServerTracker.close();
+ initializeTracker(name);
+
+ // change the state of the index and update the metadata
+ try {
+ state = IndexState.ACTIVE;
+ updateIndexMetadata();
+ } catch (IndexException e) {
+ logger.error("Failed to set the state while reindexing the index: {}", name, e);
+ return;
+ }
+
+ // start update checker
+ startIndexingSourceCheckThread();
+ logger.info("Reindexing of Semantic Index: {} has completed successfully", name);
+ }
+
+ private IndexMetadata createTemporarySolrCore() throws IndexManagementException {
+ // determine a temporary name
+ String coreName = name;
+ int count = 1;
+ do {
+ coreName = name + "-" + count;
+ count++;
+ } while ((managedSolrServer.isManagedIndex(coreName)));
+
+ return solrSemanticIndexFactory.createSolrCore(coreName, ldPathProgram);
+ }
+
+ private long indexDocuments() throws StoreException, IndexException {
+ ChangeSet cs;
+ long revision = Long.MIN_VALUE;
+ boolean noChange = false;
+ do {
+ cs = indexingSource.changes(indexingSource.getEpoch(), revision, batchSize);
+ Iterator<String> changedItr = cs.iterator();
+ while (changedItr.hasNext()) {
+ String changed = changedItr.next();
+ ContentItem ci = indexingSource.get(changed);
+ if (ci == null) {
+ performRemove(changed);
+ } else {
+ performIndex(ci);
+ }
+ }
+ noChange = cs.iterator().hasNext() ? false : true;
+ if (!noChange) {
+ revision = cs.toRevision();
+ }
+ } while (!noChange);
+ return revision;
+ }
+ }
+}
\ No newline at end of file
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndexFactory.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndexFactory.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndexFactory.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndexFactory.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.index.solr;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.Service;
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndex;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndexManager;
+import org.apache.stanbol.commons.solr.managed.IndexMetadata;
+import org.apache.stanbol.commons.solr.managed.ManagedSolrServer;
+import org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex;
+import org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndexFactory;
+import org.apache.stanbol.contenthub.index.SemanticIndexMetadataManager;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteManager;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.Constants;
+import org.osgi.framework.ServiceReference;
+import org.osgi.service.cm.Configuration;
+import org.osgi.service.cm.ConfigurationAdmin;
+import org.osgi.service.component.ComponentContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+/**
+ * This class provides facilities to create {@link SolrSemanticIndex} instances based on given LDPath program
+ * other parameters. {@link SemanticIndex}es can be created through both Felix Web console by configuring a
+ * new {@link SolrSemanticIndex} or through the REST/Java API by giving the necessary information to create
+ * the index.
+ *
+ * @author suat
+ * @author anil
+ *
+ */
+@Component(immediate = true)
+@Service(value = SolrSemanticIndexFactory.class)
+public class SolrSemanticIndexFactory extends AbstractLDPathSemanticIndexFactory {
+
+ private final Logger logger = LoggerFactory.getLogger(SolrSemanticIndexFactory.class);
+
+ private LDPathUtils ldPathUtils;
+
+ private BundleContext bundleContext;
+
+ @Reference(target = "(org.apache.solr.core.CoreContainer.name=contenthub)")
+ private ManagedSolrServer managedSolrServer;
+
+ @Reference
+ private SiteManager siteManager;
+
+ @Activate
+ public void activator(ComponentContext context) throws IndexManagementException {
+ this.bundleContext = context.getBundleContext();
+ ldPathUtils = new LDPathUtils(bundleContext.getBundle(), siteManager);
+ File indexMetadataDirectory = bundleContext.getDataFile(SolrSemanticIndexFactory.class.getName());
+ this.semanticIndexMetadataManager = new SemanticIndexMetadataManager(indexMetadataDirectory);
+ }
+
+ /**
+ * <p>
+ * This method creates a Solr based {@link SemanticIndex} using the given parameters. More specifically,
+ * {@code indexName} is used as the name of the Solr core and the configuration of the Solr core is
+ * adjusted according to the given {@code ldPathProgram}. See
+ * {@link LDPathUtils#createSchemaArchive(String, String)} method to investigate how the Solr schema
+ * configuration is created.
+ * </p>
+ * Other than the given parameters the following parameters are used to create the index with the
+ * presented default values:
+ * <ul>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_INDEX_CONTENT}: {@code true}</li>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_BATCH_SIZE}: 10</li>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_INDEXING_SOURCE_NAME}: contenthubFileStore</li>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_INDEXING_SOURCE_CHECK_PERIOD}: 10</li>
+ * <li>{@link AbstractLDPathSemanticIndexFactory#PROP_SOLR_CHECK_TIME}: 5</li>
+ * <li>{@link Constants#SERVICE_RANKING}: 0</li>
+ * </ul>
+ *
+ * Each created {@link SemanticIndex} becomes an OSGi component and it can be obtained as a
+ * {@link ServiceReference}. It is possible to get indexes through the {@link SemanticIndexManager} or
+ * {@link BundleContext}.
+ */
+ @Override
+ public String createIndex(String indexName, String indexDescription, String ldPathProgram) throws IndexManagementException {
+ if (indexName == null || indexName.isEmpty() || ldPathProgram == null || ldPathProgram.isEmpty()) {
+ throw new IndexManagementException("Index name and LDPath program cannot be null or empty");
+ }
+
+ if (managedSolrServer.isManagedIndex(indexName)) {
+ throw new IndexManagementException(String.format("There is already an index with the name: %s",
+ indexName));
+ }
+
+ Properties props = new Properties();
+ props.put(SolrSemanticIndex.PROP_NAME, indexName);
+ props.put(SolrSemanticIndex.PROP_LD_PATH_PROGRAM, ldPathProgram);
+ props.put(SolrSemanticIndex.PROP_DESCRIPTION, indexDescription);
+ return createIndex(props);
+ }
+
+ /**
+ * <p>
+ * This method creates a Solr based {@link SemanticIndex} using the parameters passed in the given
+ * {@code indexMetadata}. As specified in the
+ * {@link AbstractLDPathSemanticIndexFactory#createIndex(Properties)}, the following parameters are
+ * mandatory.
+ * <ul>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_LD_PATH_PROGRAM}</li>
+ * <li>{@link SemanticIndex#PROP_NAME}</li>
+ * </ul>
+ * {@link SemanticIndex#PROP_NAME} is used as the name of the Solr core and the configuration of the Solr
+ * core is adjusted according to the given {@link AbstractLDPathSemanticIndex#PROP_LD_PATH_PROGRAM}. See
+ * {@link LDPathUtils#createSchemaArchive(String, String)} method to investigate how the Solr schema
+ * configuration is created.
+ * </p>
+ * <p>
+ * If they are not already specified the following parameters are used to create the index with the
+ * presented default values. Different values for these parameters can also be passed in the
+ * {@code indexMetadata}:
+ * </p>
+ * <ul>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_INDEX_CONTENT}: {@code true}</li>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_BATCH_SIZE}: 10</li>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_INDEXING_SOURCE_NAME}: contenthubFileStore</li>
+ * <li>{@link AbstractLDPathSemanticIndex#PROP_INDEXING_SOURCE_CHECK_PERIOD}: 10</li>
+ * <li>{@link AbstractLDPathSemanticIndexFactory#PROP_SOLR_CHECK_TIME}: 5</li>
+ * <li>{@link Constants#SERVICE_RANKING}: 0</li>
+ * </ul>
+ *
+ * Each created {@link SemanticIndex} becomes an OSGi component and it can be obtained as a
+ * {@link ServiceReference}. It is possible to get indexes through the {@link SemanticIndexManager} or
+ * {@link BundleContext}.
+ */
+ @Override
+ public String createIndex(Properties indexMetadata) throws IndexManagementException {
+ // validate properties
+ String indexName = (String) indexMetadata.get(SemanticIndex.PROP_NAME);
+ String ldPathProgram = (String) indexMetadata.get(SolrSemanticIndex.PROP_LD_PATH_PROGRAM);
+ String configurationPID = (String) indexMetadata.get(Constants.SERVICE_PID);
+
+ if (indexName == null || indexName.isEmpty() || ldPathProgram == null || ldPathProgram.isEmpty()) {
+ throw new IndexManagementException("Index name and LDPath program cannot be null or empty");
+ }
+
+ if (managedSolrServer.isManagedIndex(indexName)) {
+ throw new IndexManagementException(String.format("There is already an index with the name: %s",
+ indexName));
+ }
+
+ checkValues(indexMetadata);
+
+ // create solr core
+ createSolrCore(indexName, ldPathProgram);
+ logger.info("Solr core for the Semantic Index: {} has been created successfully", indexName);
+
+ // activate the OSGI component if it is not already done. This is the case when an SolrSemanticIndex
+ // is created through the REST/Java API
+ if (configurationPID == null) {
+ ServiceReference reference = bundleContext
+ .getServiceReference(ConfigurationAdmin.class.getName());
+ ConfigurationAdmin configAdmin = (ConfigurationAdmin) bundleContext.getService(reference);
+ Configuration config;
+ try {
+ config = configAdmin.createFactoryConfiguration(SolrSemanticIndex.class.getName(), null);
+ configurationPID = config.getPid();
+
+ semanticIndexMetadataManager.storeIndexMetadata(configurationPID, indexMetadata);
+ config.update(indexMetadata);
+ logger.info(
+ "A new configuration has been created for the Semantic Index: {} and its metadata was stored",
+ indexName);
+ } catch (IOException e) {
+ logger.error("Failed to create Factory Configuration for SolrSemanticIndex: {}", indexName);
+ throw new IndexManagementException(String.format(
+ "Failed to create Factory Configuration for SolrSemanticIndex: %s", indexName), e);
+ }
+
+ // the index created through the Felix Web console
+ } else {
+ semanticIndexMetadataManager.storeIndexMetadata(configurationPID, indexMetadata);
+ logger.info(
+ "A configuration has already been created for the Semantic Index: {}, so only its metadata was stored",
+ indexName);
+ }
+ return configurationPID;
+ }
+
+ private void checkValues(Properties indexMetadata) {
+ if (indexMetadata.get(SolrSemanticIndex.PROP_DESCRIPTION) == null) {
+ indexMetadata.put(SolrSemanticIndex.PROP_DESCRIPTION, "");
+ }
+ if (indexMetadata.get(SolrSemanticIndex.PROP_INDEX_CONTENT) == null) {
+ indexMetadata.put(SolrSemanticIndex.PROP_INDEX_CONTENT, true);
+ }
+ if (indexMetadata.get(SolrSemanticIndex.PROP_BATCH_SIZE) == null) {
+ indexMetadata.put(SolrSemanticIndex.PROP_BATCH_SIZE, 10);
+ }
+ if (indexMetadata.get(SolrSemanticIndex.PROP_INDEXING_SOURCE_NAME) == null) {
+ indexMetadata.put(SolrSemanticIndex.PROP_INDEXING_SOURCE_NAME, "contenthubFileStore");
+ }
+ if (indexMetadata.get(SolrSemanticIndex.PROP_INDEXING_SOURCE_CHECK_PERIOD) == null) {
+ indexMetadata.put(SolrSemanticIndex.PROP_INDEXING_SOURCE_CHECK_PERIOD, 10);
+ }
+ if (indexMetadata.get(SolrSemanticIndex.PROP_SOLR_CHECK_TIME) == null) {
+ indexMetadata.put(SolrSemanticIndex.PROP_SOLR_CHECK_TIME, 5);
+ }
+ if (indexMetadata.get(Constants.SERVICE_RANKING) == null) {
+ indexMetadata.put(Constants.SERVICE_RANKING, 0);
+ }
+ }
+
+ /**
+ * Create a Solr core using the given {@code coreName} and {@code ldPathProgram}. A solr schema
+ * configuration archive is formed based on the given parameters through the
+ * {@link LDPathUtils#createSchemaArchive(String, String)} and this archive is used to create a Solr core
+ * by {@link ManagedSolrServer}.
+ *
+ * @param coreName
+ * Name of the Solr core to be created
+ * @param ldPathProgram
+ * LDPath program to adjust the Solr core configuration
+ * @return the {@link IndexMetadata} of the created Solr core
+ * @throws IndexManagementException
+ */
+ public IndexMetadata createSolrCore(String coreName, String ldPathProgram) throws IndexManagementException {
+ ArchiveInputStream coreArchive = ldPathUtils.createSchemaArchive(coreName, ldPathProgram);
+ try {
+ return managedSolrServer.createSolrIndex(coreName, coreArchive);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ throw new IndexManagementException(e.getMessage(), e);
+ } catch (SAXException e) {
+ String msg = "ManagedSolrServer cannot parse the related XML files.";
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+ }
+
+ @Override
+ public void removeIndex(String pid) throws IndexManagementException {
+
+ Properties indexMetadata = semanticIndexMetadataManager.removeIndexMetadata(pid);
+ managedSolrServer.removeIndex(indexMetadata.getProperty(SemanticIndex.PROP_NAME), true);
+ ServiceReference reference = bundleContext.getServiceReference(ConfigurationAdmin.class.getName());
+ ConfigurationAdmin configAdmin = (ConfigurationAdmin) bundleContext.getService(reference);
+ Configuration config;
+ String indexName = (String) indexMetadata.get(SemanticIndex.PROP_NAME);
+ try {
+ config = configAdmin.getConfiguration(pid);
+ config.delete();
+ logger.info("Semantic Index: {} was removed successfully", indexName);
+ } catch (IOException e) {
+ throw new IndexManagementException(String.format(
+ "Failed to remove configuration for the Semantic Index: %s", indexName));
+ }
+ }
+}
Modified: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1387977&r1=1387976&r2=1387977&view=diff
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/resources/OSGI-INF/metatype/metatype.properties (original)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/resources/OSGI-INF/metatype/metatype.properties Thu Sep 20 11:22:16 2012
@@ -16,32 +16,38 @@
#===============================================================================
#Properties and Options used to configure
#===============================================================================
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.name=Apache Stanbol Contenthub LD Path Semantic Index Configuration
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.description=Creates the underlying Solr core by parsing the provided LDPath program
-
+#SemanticIndex
Semantic-Index-Name.name=Name
-Semantic-Index-Name.description=The name identifying the index
+Semantic-Index-Name.description=The name identifying the Semantic Index
Semantic-Index-Description.name=Description
-Semantic-Index-Description.description=Description of the index
-
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.ldPathProgram.name=LDPath Program
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.ldPathProgram.description=LDPath program that will be used as a source to create the semantic index. Index fields and Solr specific configurations regarding those index fields are given in this parameter.
+Semantic-Index-Description.description=Description of the Semantic Index
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.indexContent.name=Index Content
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.indexContent.description=Set to TRUE to enable indexing content
+#AbstractLDPathSemanticIndex
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.name=Apache Stanbol Contenthub Abstract LD Path Semantic Index Configuration
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.description=AbstractLDPathSemanticIndex is an abstract class on which new LDPath based SemanticIndex implementations can be developed.
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.batchSize.name=Batch Size
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.batchSize.description=Maximum number of changes to be returned
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.ldPathProgram.name=LDPath Program
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.ldPathProgram.description=LDPath program to be used in the Semantic Index. Note that the LDPath program can be used for different purposes in different implementations. Please see the implementation specific documentation.
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.storeName.name=Store Name
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.storeName.description=Name of the Store instance to be checked for updates.
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.batchSize.name=Batch Size
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.batchSize.description=Maximum number of changes to be returned
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.storeCheckPeriod.name=Store Check Period
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.storeCheckPeriod.description=Time to check changes in the Contenthub Store in second units
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.indexingSourceName.name=Indexing Source Name
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.indexingSourceName.description=Name of the Store instance to be checked for updates.
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.solrCheckTime.name=Solr Server Check Time
-org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex.solrCheckTime.description=Maximum time in seconds to wait for the availability of the Solr Server
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.indexingSourceCheckPeriod.name=Indexing Source Check Period
+org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.indexingSourceCheckPeriod.description=Time to check changes in the specified Store in second units
service.ranking.name=Service Ranking
-service.ranking.description=To be able to use other SemanticIndex implementations rather than this, Service Ranking property of other implementations should be set higher than of this one
\ No newline at end of file
+service.ranking.description=To be able to use other specific instances of SemanticIndex implementations, Service Ranking property of instances can be set to be able obtain the custom instance
+
+#SolrSemanticIndex
+org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.name=Apache Stanbol Contenthub Solr Based Semantic Index Configuration
+org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.description=SolrSemanticIndex is an SemanticIndex implementation which basically configures the underlying Solr core by considering the specified LDPath as configuration file. ContentItem itself and named entities of contained within ContentItems to be indexed into this index are queried with the configured LDPath and gathered information is also indexed.
+
+org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.solrCheckTime.name=Solr Server Check Time
+org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.solrCheckTime.description=Maximum time in seconds to wait for the availability of the Solr Server
+
+org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.indexContent.name=Index Content
+org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex.indexContent.description=Set to TRUE to enable indexing content
\ No newline at end of file
Modified: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/revisionmanager/src/main/java/org/apache/stanbol/contenthub/revisionmanager/DerbyDBManager.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/revisionmanager/src/main/java/org/apache/stanbol/contenthub/revisionmanager/DerbyDBManager.java?rev=1387977&r1=1387976&r2=1387977&view=diff
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/revisionmanager/src/main/java/org/apache/stanbol/contenthub/revisionmanager/DerbyDBManager.java (original)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/revisionmanager/src/main/java/org/apache/stanbol/contenthub/revisionmanager/DerbyDBManager.java Thu Sep 20 11:22:16 2012
@@ -60,8 +60,6 @@ public class DerbyDBManager {
private static int MAX_ID_LENGTH = 1024;
- // private String SELECT_CHANGES =
- // "SELECT id, revision FROM \"%s\" WHERE revision > ? ORDER BY revision ASC";
private String SELECT_CHANGES = "SELECT id, revision FROM \"%s\" WHERE revision > ? AND revision <= ? ORDER BY revision ASC OFFSET ? ROWS";
private String SELECT_REVISION = "SELECT id,revision FROM \"%s\" content_item_revision WHERE id = ?";
@@ -86,7 +84,7 @@ public class DerbyDBManager {
protected void activate(ComponentContext componentContext) throws RevisionManagerException {
System.setProperty("derby.language.statementCacheSize", "0");
String stanbolHome = componentContext.getBundleContext().getProperty("sling.home");
- DB_URL = "jdbc:derby:" + stanbolHome + "/contenthub/store/revisions;create=true";
+ DB_URL = "jdbc:derby:" + stanbolHome + "/contenthub/derby/revisions;create=true";
// initialize the epoch table
createEpochTable();
}
@@ -327,7 +325,7 @@ public class DerbyDBManager {
throw new RevisionManagerException(String.format(
"There is not an epoch record for the Store: %s", storeID));
}
- log.info("The epoch for the Store: %s has been fetched as %s", storeID, epoch);
+ log.info("The epoch for the Store: {} has been fetched as {}", storeID, epoch);
} catch (SQLException e) {
log.error("Failed to get epoch of the Store: {}", storeID, e);
@@ -566,8 +564,8 @@ public class DerbyDBManager {
PreparedStatement ps = null;
try {
ps = con.prepareStatement(String.format(UPDATE_REVISION, tableName));
- ps.setString(1, itemID);
- ps.setLong(2, revision);
+ ps.setString(2, itemID);
+ ps.setLong(1, revision);
ps.executeUpdate();
log.info("Revision of the item: %s in table: %s has been updated as %d", new Object[] {itemID,
tableName,
Modified: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/featured/src/main/java/org/apache/stanbol/contenthub/search/featured/FeaturedSearchImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/featured/src/main/java/org/apache/stanbol/contenthub/search/featured/FeaturedSearchImpl.java?rev=1387977&r1=1387976&r2=1387977&view=diff
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/featured/src/main/java/org/apache/stanbol/contenthub/search/featured/FeaturedSearchImpl.java (original)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/featured/src/main/java/org/apache/stanbol/contenthub/search/featured/FeaturedSearchImpl.java Thu Sep 20 11:22:16 2012
@@ -42,7 +42,7 @@ import org.apache.solr.common.params.Sol
import org.apache.stanbol.commons.semanticindex.index.IndexException;
import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
import org.apache.stanbol.commons.semanticindex.index.SemanticIndexManager;
-import org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex;
+import org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex;
import org.apache.stanbol.contenthub.search.solr.util.SolrQueryUtil;
import org.apache.stanbol.contenthub.servicesapi.index.search.SearchException;
import org.apache.stanbol.contenthub.servicesapi.index.search.featured.ConstrainedDocumentSet;
@@ -190,9 +190,9 @@ public class FeaturedSearchImpl implemen
@Override
public List<FacetResult> getAllFacetResults(String indexName) throws SearchException {
- LDPathSemanticIndex semanticIndex = null;
+ SolrSemanticIndex semanticIndex = null;
try {
- semanticIndex = (LDPathSemanticIndex) semanticIndexManager.getIndex(indexName);
+ semanticIndex = (SolrSemanticIndex) semanticIndexManager.getIndex(indexName);
} catch (IndexManagementException e) {
log.error("Failed to get index {}", indexName, e);
throw new SearchException("Failed to get index " + indexName, e);
Modified: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/solr/src/main/java/org/apache/stanbol/contenthub/search/solr/SolrSearchImpl.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/solr/src/main/java/org/apache/stanbol/contenthub/search/solr/SolrSearchImpl.java?rev=1387977&r1=1387976&r2=1387977&view=diff
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/solr/src/main/java/org/apache/stanbol/contenthub/search/solr/SolrSearchImpl.java (original)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/solr/src/main/java/org/apache/stanbol/contenthub/search/solr/SolrSearchImpl.java Thu Sep 20 11:22:16 2012
@@ -29,7 +29,7 @@ import org.apache.solr.common.params.Sol
import org.apache.stanbol.commons.semanticindex.index.IndexException;
import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
import org.apache.stanbol.commons.semanticindex.index.SemanticIndexManager;
-import org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex;
+import org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex;
import org.apache.stanbol.contenthub.search.solr.util.SolrQueryUtil;
import org.apache.stanbol.contenthub.servicesapi.index.search.SearchException;
import org.apache.stanbol.contenthub.servicesapi.index.search.solr.SolrSearch;
@@ -53,7 +53,7 @@ public class SolrSearchImpl implements S
SolrQuery solrQuery = null;
SolrServer solrServer = null;
try {
- LDPathSemanticIndex semanticIndex = (LDPathSemanticIndex) semanticIndexManager
+ SolrSemanticIndex semanticIndex = (SolrSemanticIndex) semanticIndexManager
.getIndex(indexName);
solrServer = semanticIndex.getServer();
solrQuery = SolrQueryUtil.prepareSolrQuery(solrServer, queryTerm);
@@ -77,7 +77,7 @@ public class SolrSearchImpl implements S
public QueryResponse search(SolrParams solrQuery, String indexName) throws SearchException {
SolrServer solrServer = null;
try {
- LDPathSemanticIndex semanticIndex = (LDPathSemanticIndex) semanticIndexManager
+ SolrSemanticIndex semanticIndex = (SolrSemanticIndex) semanticIndexManager
.getIndex(indexName);
solrServer = semanticIndex.getServer();
} catch (IndexManagementException e) {
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/SolrSemanticIndexFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/SolrSemanticIndexFactoryTest.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/SolrSemanticIndexFactoryTest.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/SolrSemanticIndexFactoryTest.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.test.index.solr;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.sling.junit.annotations.SlingAnnotationsTestRunner;
+import org.apache.sling.junit.annotations.TestReference;
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndex;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndexManager;
+import org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex;
+import org.apache.stanbol.contenthub.index.solr.SolrSemanticIndexFactory;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.osgi.framework.BundleContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@RunWith(SlingAnnotationsTestRunner.class)
+public class SolrSemanticIndexFactoryTest {
+ private static Logger logger = LoggerFactory.getLogger(SolrSemanticIndexFactoryTest.class);
+
+ @TestReference
+ private SolrSemanticIndexFactory solrSemanticIndexFactory;
+
+ @TestReference
+ private SemanticIndexManager semanticIndexManager;
+
+ @TestReference
+ private BundleContext bundleContext;
+
+ @Test
+ public void solrSemanticIndexFactoryTest() {
+ assertNotNull("Expecting SolrSemanticIndexFactory to be injected by Sling test runner",
+ solrSemanticIndexFactory);
+ }
+
+ @Test
+ public void semanticIndexManagerTest() {
+ assertNotNull("Expecting SemanticIndexManager to be injected by Sling test runner",
+ semanticIndexManager);
+ }
+
+ @Test
+ public void bundleContextTest() {
+ assertNotNull("Expecting BundleContext to be injected by Sling test runner", bundleContext);
+ }
+
+ @Test
+ public void testCreateIndexWithoutName() {
+ Properties indexMetadata = new Properties();
+ indexMetadata
+ .put(SolrSemanticIndex.PROP_LD_PATH_PROGRAM,
+ "@prefix dbp-ont : <http://dbpedia.org/ontology/>; city = dbp-ont:city / rdfs:label :: xsd:string; ");
+ try {
+ solrSemanticIndexFactory.createIndex(indexMetadata);
+ assertTrue("An Index cannot be created without name", false);
+ } catch (IndexManagementException e) {
+
+ }
+ }
+
+ @Test
+ public void testCreateIndexWithoutProgram() {
+ Properties indexMetadata = new Properties();
+ indexMetadata.put(SemanticIndex.PROP_NAME, "test_index_name");
+ try {
+ solrSemanticIndexFactory.createIndex(indexMetadata);
+ assertTrue("An Index cannot be created without program", false);
+ } catch (IndexManagementException e) {
+
+ }
+ }
+
+ @Test
+ public void testCreateRemoveIndex() throws IndexManagementException, InterruptedException {
+ String name = "test_index_name";
+ String program = "@prefix dbp-ont : <http://dbpedia.org/ontology/>; city = dbp-ont:city / rdfs:label :: xsd:string;";
+ Properties indexMetadata = new Properties();
+ indexMetadata.put(SemanticIndex.PROP_NAME, name);
+ indexMetadata.put(SolrSemanticIndex.PROP_LD_PATH_PROGRAM, program);
+ String pid = solrSemanticIndexFactory.createIndex(indexMetadata);
+
+ // retrieve SolrSemanticIndex
+ SolrSemanticIndex semanticIndex;
+ String indexMetadataFilePath;
+ File indexMetadataDirectory;
+ File file;
+ try {
+ semanticIndex = (SolrSemanticIndex) semanticIndexManager.getIndex(name);
+ int timeoutCount = 0;
+ while (semanticIndex == null) {
+ if (timeoutCount == 8) break;
+ Thread.sleep(500);
+ semanticIndex = (SolrSemanticIndex) semanticIndexManager.getIndex(name);
+ timeoutCount++;
+ }
+ assertNotNull("Failed to create SolrSemanticIndex with name " + name, semanticIndex);
+
+ // check IndexMetadata folder exists
+ indexMetadataDirectory = bundleContext
+ .getServiceReference(SolrSemanticIndexFactory.class.getName()).getBundle()
+ .getBundleContext().getDataFile(SolrSemanticIndexFactory.class.getName());
+ assertTrue("IndexMetadata Directory does not exist", indexMetadataDirectory.exists());
+
+ // check IndexMetadata files of indexes before remove index
+ indexMetadataFilePath = indexMetadataDirectory.getAbsolutePath() + File.separator + pid
+ + ".props";
+ file = new File(indexMetadataFilePath);
+ assertTrue("IndexMetadata File cannot be found for pid: " + pid, file.exists());
+
+ } finally {
+ // remove SolrSemanticIndex
+ solrSemanticIndexFactory.removeIndex(pid);
+ }
+
+ // wait some time to let OSGi remove the configuration
+ int timeoutCount = 0;
+ semanticIndex = (SolrSemanticIndex) semanticIndexManager.getIndex(name);
+ while (semanticIndex != null) {
+ if (timeoutCount == 8) break;
+ Thread.sleep(500);
+ semanticIndex = (SolrSemanticIndex) semanticIndexManager.getIndex(name);
+ timeoutCount++;
+ }
+
+ assertNull(String.format("SolrSemanticIndex with name %s cannot be removed", name), semanticIndex);
+ if (semanticIndex == null) {
+ assertFalse("SemanticIndex is removed, but still appears configured.", solrSemanticIndexFactory
+ .getSemanticIndexMetadataManager().isConfigured(pid));
+ }
+
+ // check IndexMetadata files of indexes after remove index
+ indexMetadataFilePath = indexMetadataDirectory.getAbsolutePath() + File.separator + pid + ".props";
+ file = new File(indexMetadataFilePath);
+ assertFalse("IndexMetadata File cannot be removed for pid: " + pid, file.exists());
+ }
+
+ @Test
+ public void testIndexMetaData() throws IndexManagementException, InterruptedException {
+
+ String name1 = "test_index_name1";
+ String name2 = "test_index_name2";
+ String program = "@prefix dbp-ont : <http://dbpedia.org/ontology/>; city = dbp-ont:city / rdfs:label :: xsd:string;";
+
+ Properties indexMetadata = new Properties();
+ indexMetadata.put(SemanticIndex.PROP_NAME, name1);
+ indexMetadata.put(SolrSemanticIndex.PROP_LD_PATH_PROGRAM, program);
+ String pid1 = solrSemanticIndexFactory.createIndex(indexMetadata);
+ String pid2 = solrSemanticIndexFactory.createIndex(name2, "Test Index Description", program);
+
+ // create and retrieve SolrSemanticIndex
+ SolrSemanticIndex semanticIndex1 = (SolrSemanticIndex) semanticIndexManager.getIndex(name1);
+ SolrSemanticIndex semanticIndex2 = (SolrSemanticIndex) semanticIndexManager.getIndex(name2);
+ int timeoutCount = 0;
+ while (semanticIndex1 == null || semanticIndex2 == null) {
+ if (timeoutCount == 8) break;
+ Thread.sleep(500);
+ semanticIndex1 = (SolrSemanticIndex) semanticIndexManager.getIndex(name1);
+ semanticIndex2 = (SolrSemanticIndex) semanticIndexManager.getIndex(name2);
+ timeoutCount++;
+ }
+
+ File indexMetadataDirectory = bundleContext
+ .getServiceReference(SolrSemanticIndexFactory.class.getName()).getBundle()
+ .getBundleContext().getDataFile(SolrSemanticIndexFactory.class.getName());
+
+ Map<String,Properties> indexMetadataMap = new HashMap<String,Properties>();
+ // load index metadata to memory
+ File[] metadataList = indexMetadataDirectory.listFiles();
+ for (File configFile : metadataList) {
+ String pid = configFile.getName().substring(0, configFile.getName().lastIndexOf('.'));
+ Properties props = new Properties();
+ InputStream is = null;
+ try {
+ is = new FileInputStream(configFile);
+ props.load(is);
+ indexMetadataMap.put(pid, props);
+ logger.info("Index metadata has been loaded from the location: {}",
+ configFile.getAbsolutePath());
+ } catch (FileNotFoundException e) {
+ logger.error("IndexMetadata file cannot be found");
+ throw new IndexManagementException("IndexMetadata file cannot be found");
+ } catch (IOException e) {
+ logger.error("Failed to read from input stream");
+ throw new IndexManagementException("Failed to read from input stream");
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ }
+
+ // test getAllIndexMetadata
+ Map<String,Properties> allIndexMetadata = solrSemanticIndexFactory
+ .getSemanticIndexMetadataManager().getAllIndexMetadata();
+ assertTrue("getAllIndexMetadata() cannot return the expected value",
+ allIndexMetadata.equals(indexMetadataMap));
+
+ // check IndexMetadata map entries
+ assertTrue(
+ String.format("IndexMetadata for %s is not match with the expected value", pid1),
+ solrSemanticIndexFactory.getSemanticIndexMetadataManager().getIndexMetadata(pid1)
+ .equals(indexMetadataMap.get(pid1)));
+ assertTrue(
+ String.format("IndexMetadata for %s is not match with the expected value", pid1),
+ solrSemanticIndexFactory.getSemanticIndexMetadataManager().getIndexMetadata(pid2)
+ .equals(indexMetadataMap.get(pid2)));
+
+ // update indexMetadataMap
+ indexMetadataMap.get(pid1).put("test field", "test value");
+ solrSemanticIndexFactory.getSemanticIndexMetadataManager().updateIndexMetadata(pid1,
+ indexMetadataMap.get(pid1));
+ assertTrue("IndexMetadata cannot be updated properly", solrSemanticIndexFactory
+ .getSemanticIndexMetadataManager().getIndexMetadata(pid1).equals(indexMetadataMap.get(pid1)));
+
+ // remove SolrSemanticIndex
+ solrSemanticIndexFactory.removeIndex(pid1);
+ solrSemanticIndexFactory.removeIndex(pid2);
+ }
+}