You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by su...@apache.org on 2012/09/20 13:22:17 UTC
svn commit: r1387977 [1/3] - in
/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub:
defaults/src/main/resources/config/
index/src/main/java/org/apache/stanbol/contenthub/index/
index/src/main/java/org/apache/stanbol/contenthub/inde...
Author: suat
Date: Thu Sep 20 11:22:16 2012
New Revision: 1387977
URL: http://svn.apache.org/viewvc?rev=1387977&view=rev
Log:
STANBOL499:
-Implemented an Abstract class (AbstractLDPathSemanticIndex) on which different SemanticIndex implementations can be developed
-Renamed the old LDPathSemanticIndex to SolrSemanticIndex
-Implemented a simple factory to enable creation and deletion of indexes. In the future this can be merged with SemanticIndexManager interface
Added:
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/defaults/src/main/resources/config/org.apache.stanbol.contenthub.index.solr.SolrSemanticIndex-default.config
- copied unchanged from r1383419, incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/defaults/src/main/resources/config/org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex-default.config
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndex.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndexFactory.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/SemanticIndexMetadataManager.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/LDPathUtils.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndex.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/SolrSemanticIndexFactory.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/SolrSemanticIndexFactoryTest.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/solr/SolrSemanticIndexTest.java
Removed:
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/defaults/src/main/resources/config/org.apache.stanbol.contenthub.index.ldpath.LDPathSemanticIndex-default.config
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/ldpath/
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/test/src/main/java/org/apache/stanbol/contenthub/test/index/ldpath/
Modified:
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/resources/OSGI-INF/metatype/metatype.properties
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/revisionmanager/src/main/java/org/apache/stanbol/contenthub/revisionmanager/DerbyDBManager.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/featured/src/main/java/org/apache/stanbol/contenthub/search/featured/FeaturedSearchImpl.java
incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/search/solr/src/main/java/org/apache/stanbol/contenthub/search/solr/SolrSearchImpl.java
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndex.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndex.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndex.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndex.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,507 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.index;
+
+import java.io.StringReader;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Collection;
+import java.util.Dictionary;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.clerezza.rdf.core.Resource;
+import org.apache.felix.scr.annotations.Properties;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.stanbol.commons.semanticindex.index.IndexException;
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.commons.semanticindex.index.IndexState;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndex;
+import org.apache.stanbol.commons.semanticindex.store.ChangeSet;
+import org.apache.stanbol.commons.semanticindex.store.EpochException;
+import org.apache.stanbol.commons.semanticindex.store.IndexingSource;
+import org.apache.stanbol.commons.semanticindex.store.StoreException;
+import org.apache.stanbol.contenthub.index.solr.LDPathUtils;
+import org.apache.stanbol.enhancer.ldpath.EnhancerLDPath;
+import org.apache.stanbol.enhancer.ldpath.backend.ContentItemBackend;
+import org.apache.stanbol.enhancer.servicesapi.ContentItem;
+import org.apache.stanbol.entityhub.core.model.InMemoryValueFactory;
+import org.apache.stanbol.entityhub.core.utils.OsgiUtils;
+import org.apache.stanbol.entityhub.ldpath.EntityhubLDPath;
+import org.apache.stanbol.entityhub.ldpath.backend.SiteManagerBackend;
+import org.apache.stanbol.entityhub.servicesapi.model.Representation;
+import org.apache.stanbol.entityhub.servicesapi.model.ValueFactory;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteManager;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.Constants;
+import org.osgi.framework.InvalidSyntaxException;
+import org.osgi.framework.ServiceReference;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.ComponentException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import at.newmedialab.ldpath.LDPath;
+import at.newmedialab.ldpath.exception.LDPathParseException;
+import at.newmedialab.ldpath.model.programs.Program;
+
+/**
+ * <p>
+ * Abstract class which can be used to develop new OSGi based {@link SemanticIndex} implementations. It
+ * provides various kind of methods which are possibly used by different extenstion of this class. By default
+ * it implements the {@link SemanticIndex} interface. However, not all of the methods are implemented. So, new
+ * implementations are expected to implement the missing ones.
+ * </p>
+ * <p>
+ * For the provided methods please refer to the specific documentations of methods below.
+ * </p>
+ *
+ * @author meric
+ * @author suat
+ *
+ */
+@Properties(value = {
+ @Property(name = SemanticIndex.PROP_NAME),
+ @Property(name = SemanticIndex.PROP_DESCRIPTION),
+ @Property(name = AbstractLDPathSemanticIndex.PROP_LD_PATH_PROGRAM),
+ @Property(name = AbstractLDPathSemanticIndex.PROP_BATCH_SIZE, intValue = 10),
+ @Property(name = AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME, value = "contenthubFileStore"),
+ @Property(name = AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_CHECK_PERIOD, intValue = 10),
+ @Property(name = Constants.SERVICE_RANKING, intValue = 0)})
+public abstract class AbstractLDPathSemanticIndex implements SemanticIndex<ContentItem> {
+
+ private static final Logger logger = LoggerFactory.getLogger(AbstractLDPathSemanticIndex.class);
+
+ public static final String PROP_LD_PATH_PROGRAM = "org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.ldPathProgram";
+ public static final String PROP_INDEXING_SOURCE_CHECK_PERIOD = "org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.indexingSourceCheckPeriod";
+ public static final String PROP_INDEXING_SOURCE_NAME = "org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.indexingSourceName";
+ public static final String PROP_BATCH_SIZE = "org.apache.stanbol.contenthub.index.AbstractLDPathSemanticIndex.batchSize";
+
+ protected String name;
+ protected String description;
+ protected String ldPathProgram;
+ protected Program<Object> objectProgram;
+ protected int batchSize;
+ protected int indexingSourceCheckPeriod;
+ protected long epoch;
+ protected long revision = Long.MIN_VALUE;
+ protected volatile IndexState state = IndexState.UNINIT;
+ protected String pid;
+ protected ComponentContext componentContext;
+ private Integer indexingCount;
+ // store update check thread
+ private Thread pollingThread;
+ private volatile Boolean deactivate = new Boolean(false);
+ // reindexer thread
+ protected Thread reindexerThread;
+ protected IndexingSource<ContentItem> indexingSource;
+ protected SemanticIndexMetadataManager semanticIndexMetadataManager;
+
+ @Reference
+ protected SiteManager siteManager;
+
+ /**
+ * Updates metadata of the index persistently
+ *
+ * @throws IndexException
+ */
+ protected abstract void updateIndexMetadata() throws IndexException;
+
+ /**
+ * Starts the reindexing operation within the index. This situation occurs when the LDPath of the index
+ * changes.
+ */
+ protected abstract void startReindexing();
+
+ /**
+ * Initializes the common properties to be obtained from the {@link ComponentContext}. Furthermore, this
+ * method tries to fetch the specified {@link IndexingSource} from the OSGi environment. This method is
+ * expected to be called in the {@code activate} method of the actual implementation of this abstract
+ * class.
+ *
+ * @param componentContext
+ * {@link ComponentException} of the actual implementation of this abstract class
+ * @throws ConfigurationException
+ * @throws IndexException
+ * @throws IndexManagementException
+ * @throws StoreException
+ */
+ protected void activate(ComponentContext componentContext) throws ConfigurationException,
+ IndexException,
+ IndexManagementException,
+ StoreException {
+ @SuppressWarnings("rawtypes")
+ Dictionary properties = componentContext.getProperties();
+ this.name = (String) OsgiUtils.checkProperty(properties, SemanticIndex.PROP_NAME);
+ this.ldPathProgram = (String) OsgiUtils.checkProperty(properties,
+ AbstractLDPathSemanticIndex.PROP_LD_PATH_PROGRAM);
+ this.description = (String) OsgiUtils.checkProperty(properties, SemanticIndex.PROP_DESCRIPTION);
+ this.batchSize = (Integer) OsgiUtils.checkProperty(properties,
+ AbstractLDPathSemanticIndex.PROP_BATCH_SIZE);
+ this.indexingSourceCheckPeriod = (Integer) OsgiUtils.checkProperty(properties,
+ AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_CHECK_PERIOD);
+ this.componentContext = componentContext;
+ this.indexingCount = 0;
+ this.pid = (String) properties.get(Constants.SERVICE_PID);
+
+ initializeIndexingSource((String) OsgiUtils.checkProperty(properties,
+ AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME), componentContext);
+ initializeLDPathProgram();
+ }
+
+ protected void deactivate(ComponentContext componentContext) {
+ // close store check thread and solr core tracker
+ deactivate = true;
+ if (pollingThread != null) {
+ pollingThread.interrupt();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void initializeIndexingSource(String indexingSourceName, ComponentContext componentContext) throws ConfigurationException {
+ BundleContext bundleContext = componentContext.getBundleContext();
+ try {
+ ServiceReference[] indexingSources = bundleContext.getServiceReferences(
+ IndexingSource.class.getName(), null);
+ for (ServiceReference serviceReference : indexingSources) {
+ Object indexingSource = bundleContext.getService(serviceReference);
+ Type[] genericInterfaces = indexingSource.getClass().getGenericInterfaces();
+ if (genericInterfaces.length == 1 && genericInterfaces[0] instanceof ParameterizedType) {
+ Type[] types = ((ParameterizedType) genericInterfaces[0]).getActualTypeArguments();
+ try {
+ @SuppressWarnings("unused")
+ Class<ContentItem> contentItemClass = (Class<ContentItem>) types[0];
+ if (((IndexingSource<ContentItem>) indexingSource).getName().equals(
+ indexingSourceName)) {
+ this.indexingSource = (IndexingSource<ContentItem>) indexingSource;
+ }
+ } catch (ClassCastException e) {
+ // ignore
+ }
+ }
+ }
+ if (this.indexingSource == null) {
+ throw new ConfigurationException(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME,
+ "There is no IndexingSource<ContentItem> for the given IndexingSource name: "
+ + indexingSourceName);
+ }
+ } catch (InvalidSyntaxException e) {
+ // ignore as there is no filter
+ }
+ }
+
+ private void initializeLDPathProgram() throws IndexException, IndexManagementException {
+ // create program for EntityhubLDPath
+ SiteManagerBackend backend = new SiteManagerBackend(siteManager);
+ ValueFactory vf = InMemoryValueFactory.getInstance();
+ EntityhubLDPath ldPath = new EntityhubLDPath(backend, vf);
+ try {
+ this.objectProgram = ldPath.parseProgram(LDPathUtils.constructReader(this.ldPathProgram));
+ } catch (LDPathParseException e) {
+ logger.error("Should never happen!!!!!", e);
+ throw new IndexException("Failed to create Program from the parsed LDPath", e);
+ }
+ }
+
+ @Override
+ public String getName() {
+ return this.name;
+ }
+
+ @Override
+ public Class<ContentItem> getIntdexType() {
+ return ContentItem.class;
+ }
+
+ @Override
+ public String getDescription() {
+ return this.description;
+ }
+
+ @Override
+ public IndexState getState() {
+ synchronized (state) {
+ return state;
+ }
+ }
+
+ @Override
+ public void persist(long revision) throws IndexException {
+ this.revision = revision;
+ updateIndexMetadata();
+ }
+
+ @Override
+ public long getRevision() {
+ return this.revision;
+ }
+
+ /**
+ * Checks whether the name of the associated {@link IndexingSource} or name of the index has been changed.
+ *
+ * @param name
+ * new name of the SemanticIndex
+ * @param indexingSourceName
+ * new name of the {@link IndexingSource} associated with this index
+ * @param oldMetadata
+ * old metadata of the SemanticIndex
+ * @return {@code true} if the LDPath program has changed, otherwise {@code false}
+ * @throws ConfigurationException
+ */
+ protected void checkUnmodifiableConfigurations(String name,
+ String indexingSourceName,
+ java.util.Properties oldMetadata) throws ConfigurationException {
+
+ // name of the semantic index has changed
+ if (!name.equals(oldMetadata.get(SemanticIndex.PROP_NAME))) {
+ throw new ConfigurationException(SemanticIndex.PROP_NAME,
+ "It is not allowed to change the name of a Semantic Index");
+ }
+
+ // name of the indexing source has changed
+ if (!indexingSourceName
+ .equals(oldMetadata.get(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME))) {
+ throw new ConfigurationException(AbstractLDPathSemanticIndex.PROP_INDEXING_SOURCE_NAME,
+ "For the time being, it is not allowed to change the name of the indexing source.");
+ }
+ }
+
+ /**
+ * Checks the reindexing condition by comparing the new current LDPath program of the index with the older
+ * one
+ *
+ * @param ldPath
+ * new LDPath program of the SemanticIndex
+ * @param oldMetadata
+ * old metadata of the SemanticIndex
+ * @return {@code true} if the LDPath has changed, otherwise {@code false}
+ */
+ protected boolean checkReindexingCondition(String ldPath, java.util.Properties oldMetadata) {
+ // ldpath of the semantic has changed, reindexing needed
+ if (!ldPath.equals(oldMetadata.get(AbstractLDPathSemanticIndex.PROP_LD_PATH_PROGRAM))) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Increases the counter e.g the {@link #indexingCount} by one. This means that a client is occupying the
+ * index for an indexing operation. For each separate request the counter is increased by one.
+ * Furthermore, this method sets the state of the index as {@link IndexState#INDEXING} and updates the
+ * metadata of the index persistently.
+ *
+ * @throws IndexException
+ */
+ protected void semUp() throws IndexException {
+ synchronized (this.indexingCount) {
+ this.indexingCount++;
+ this.state = IndexState.INDEXING;
+ updateIndexMetadata();
+ }
+ }
+
+ /**
+ * Decreases the counter e.g the {@link #indexingCount} by one. This means that a client has finished the
+ * interaction with the index. Furthermore, this method sets the state of the index as
+ * {@link IndexState#ACTIVE} and updates the metadata of the index persistently.
+ *
+ * @throws IndexException
+ */
+ protected void semDown() throws IndexException {
+ synchronized (this.indexingCount) {
+ this.indexingCount--;
+ if (this.indexingCount == 0) {
+ this.state = IndexState.ACTIVE;
+ updateIndexMetadata();
+ }
+ }
+ }
+
+ /**
+ * This method executes the LDPath program, which was used to configure this index, on the enhancements of
+ * submitted content by means of the Entityhub. In other words, additional information is gathered from
+ * the Entityhub for each entity detected in the enhancements by querying the ldpath of this index.
+ * Furthermore, the same LDPath is also executed on the given {@link ContentItem} through the
+ * {@link ContentItemBackend}.
+ *
+ * @param contexts
+ * a {@link Set} of URIs (string representations) that are used as starting nodes to execute
+ * LDPath program of this index. The context are the URIs of the entities detected in the
+ * enhancements of the content submitted.
+ * @param ci
+ * {@link ContentItem} on on which the LDPath associated with this index will be executed
+ * @return the {@link Map} containing the results obtained by executing the given program on the given
+ * contexts. Keys of the map corresponds to fields in the program and values of the map
+ * corresponds to results obtained for the field specified in the key.
+ * @throws IndexManagementException
+ */
+ protected Map<String,Collection<?>> executeProgram(Set<String> contexts, ContentItem ci) throws IndexManagementException {
+ Map<String,Collection<?>> results = new HashMap<String,Collection<?>>();
+
+ // execute the given LDPath for each context passed in contexts parameter
+ SiteManagerBackend backend = new SiteManagerBackend(siteManager);
+ ValueFactory vf = InMemoryValueFactory.getInstance();
+ EntityhubLDPath entityhubPath = new EntityhubLDPath(backend, vf);
+ Representation representation;
+ for (String context : contexts) {
+ representation = entityhubPath.execute(vf.createReference(context), this.objectProgram);
+ Iterator<String> fieldNames = representation.getFieldNames();
+ while (fieldNames.hasNext()) {
+ String fieldName = fieldNames.next();
+ Iterator<Object> valueIterator = representation.get(fieldName);
+ Set<Object> values = new HashSet<Object>();
+ while (valueIterator.hasNext()) {
+ values.add(valueIterator.next());
+ }
+ if (results.containsKey(fieldName)) {
+ @SuppressWarnings("unchecked")
+ Collection<Object> resultCollection = (Collection<Object>) results.get(fieldName);
+ Collection<Object> tmpCol = (Collection<Object>) values;
+ for (Object o : tmpCol) {
+ resultCollection.add(o);
+ }
+ } else {
+ results.put(fieldName, values);
+ }
+ }
+ }
+
+ // execute the LDPath on the given ContentItem
+ ContentItemBackend contentItemBackend = new ContentItemBackend(ci, true);
+ LDPath<Resource> resourceLDPath = new LDPath<Resource>(contentItemBackend, EnhancerLDPath.getConfig());
+ Program<Resource> resourceProgram;
+ try {
+ resourceProgram = resourceLDPath.parseProgram(new StringReader(this.ldPathProgram));
+ Map<String,Collection<?>> ciBackendResults = resourceProgram.execute(contentItemBackend,
+ ci.getUri());
+ for (Entry<String,Collection<?>> result : ciBackendResults.entrySet()) {
+ if (results.containsKey(result.getKey())) {
+ @SuppressWarnings("unchecked")
+ Collection<Object> resultsValue = (Collection<Object>) results.get(result.getKey());
+ resultsValue.addAll(result.getValue());
+ } else {
+ results.put(result.getKey(), result.getValue());
+ }
+
+ }
+ } catch (LDPathParseException e) {
+ logger.error("Failed to create Program<Resource> from the LDPath program", e);
+ }
+
+ return results;
+ }
+
+ /**
+ * Starts to poll the changes from the associated {@link IndexingSource} using a
+ * {@link IndexingSourceUpdateChecker}
+ */
+ protected void startIndexingSourceCheckThread() {
+ pollingThread = new Thread(new IndexingSourceUpdateChecker(), "IndexingSourceUpdateChecker");
+ pollingThread.start();
+ }
+
+ /**
+ * Separate thread to poll changes in the {@link IndexingSource}
+ *
+ * @author meric
+ *
+ */
+ private class IndexingSourceUpdateChecker implements Runnable {
+ @Override
+ public void run() {
+ while (!deactivate) {
+ logger.info("Pooling thread for index: {} will check the changes", name);
+ // if the polling thread is interrupted i.e the parent index component is deactivated,
+ // stop polling
+ if (Thread.currentThread().isInterrupted()) {
+ return;
+ }
+
+ ChangeSet changeSet = null;
+ try {
+ changeSet = indexingSource.changes(epoch, revision, batchSize);
+ } catch (StoreException e) {
+ logger.error(
+ String.format(
+ "Failed to get changes from FileRevisionManager with start revision: %s and batch size: %s for IndexingSource: %s",
+ revision, batchSize, indexingSource.getName()), e);
+ } catch (EpochException e) {
+ if (e.getActiveEpoch() > e.getRequestEpoch()) {
+ // epoch of the IndexingSource has increased. So, a reindexing is needed.
+ // Start the reindexing thread and terminate this one
+ logger.info(
+ "Epoch of the IndexingSource: {} has increase. So, a reindexing will be in progress",
+ indexingSource.getName());
+ epoch = e.getActiveEpoch();
+ state = IndexState.REINDEXING;
+ startReindexing();
+ return;
+ }
+ }
+ if (changeSet != null) {
+ Iterator<String> changedItems = changeSet.iterator();
+ boolean persist = true;
+ while (changedItems.hasNext()) {
+ String changedItem = changedItems.next();
+ ContentItem ci;
+ try {
+ ci = indexingSource.get(changedItem);
+ if (ci != null) {
+ index(ci);
+ logger.info("ContentItem with Uri {} is indexed to {}", changedItem, name);
+ } else {
+ remove(changedItem);
+ }
+
+ } catch (StoreException e) {
+ logger.error("Failed to retrieve contentitem with uri: {}", changedItem);
+ persist = false;
+ break;
+ } catch (IndexException e) {
+ logger.error("Failed to index contentitem with uri: {}", changedItem);
+ persist = false;
+ break;
+ }
+ }
+ if (persist) {
+ try {
+ if (changeSet.iterator().hasNext()) {
+ persist(changeSet.toRevision());
+ }
+ } catch (IndexException e) {
+ logger.error("Index revision cannot be persist", e);
+ }
+ }
+ }
+ try {
+ Thread.sleep(1000 * indexingSourceCheckPeriod);
+ } catch (InterruptedException e) {
+ logger.info(
+ "Indexing Source Checker for index: {} is interrupted while sleeping. Closing the thread",
+ name);
+ return;
+ }
+ }
+ }
+ }
+}
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndexFactory.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndexFactory.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndexFactory.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/AbstractLDPathSemanticIndexFactory.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.index;
+
+import java.util.Properties;
+
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndex;
+
+/**
+ * This is an abstract factory which can be used to build other {@link SemanticIndex} factories.
+ *
+ * @author meric
+ *
+ */
+public abstract class AbstractLDPathSemanticIndexFactory {
+ /**
+ * Manager to keep track of the metadata regarding the {@link SemanticIndex}es managed by this factory
+ */
+ protected SemanticIndexMetadataManager semanticIndexMetadataManager;
+
+ /**
+ * Returns the {@link SemanticIndexMetadataManager} associated with this factory
+ *
+ * @return
+ */
+ public SemanticIndexMetadataManager getSemanticIndexMetadataManager() {
+ if (this.semanticIndexMetadataManager != null) {
+ return this.semanticIndexMetadataManager;
+ } else {
+ throw new IllegalStateException("SemanticIndexMetadataManager has not been initialized yet");
+ }
+ }
+
+ /**
+ * Creates an {@link SemanticIndex} instance based on the given name, description and LDPath program.
+ *
+ * @param indexName
+ * name of the index to be created
+ * @param indexDescription
+ * description of the index to be created
+ * @param ldPathProgram
+ * LDPath program of the index
+ * @throws IndexManagementException
+ */
+ public abstract String createIndex(String indexName, String indexDescription, String ldPathProgram) throws IndexManagementException;
+
+ /**
+ * Creates an {@link SemanticIndex} instance based on the given index metadata. However, provided
+ * {@link Properties} must include the following mandatory items.
+ * <ul>
+ * <li><b>{@link SemanticIndex#PROP_NAME}</b></li>
+ * <li><b>{@link AbstractLDPathSemanticIndex#PROP_LD_PATH_PROGRAM}</b></li>
+ * </ul>
+ *
+ * Other parameters required for specific implementations of this method should be seen in their own
+ * documentations.
+ *
+ * @param indexMetadata
+ * {@link Properties} containing the possible metadata about the index to be created
+ * @throws IndexManagementException
+ */
+ public abstract String createIndex(Properties indexMetadata) throws IndexManagementException;
+
+ /**
+ * Remove the underlying index and all of its metadata associated with the {@code pid}
+ *
+ * @param pid
+ * persistent identifier (pid) of the factory configuration of {@link SemanticIndex}
+ * @throws IndexManagementException
+ */
+ public abstract void removeIndex(String pid) throws IndexManagementException;
+}
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/SemanticIndexMetadataManager.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/SemanticIndexMetadataManager.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/SemanticIndexMetadataManager.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/SemanticIndexMetadataManager.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.index;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.commons.semanticindex.index.SemanticIndex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Manager to keep track of metadata of {@link SemanticIndex}es. This manager stores the managed metadata
+ * persistently within file system directory specified in its constructor:
+ * {@link #SemanticIndexMetadataManager(File)}.
+ *
+ * @author suat
+ *
+ */
+public class SemanticIndexMetadataManager {
+ private final Logger logger = LoggerFactory.getLogger(SemanticIndexMetadataManager.class);
+
+ private Map<String,Properties> indexMetadataMap = new HashMap<String,Properties>();
+ private File indexMetadataDirectory;
+
+ /**
+ * Default constructor of this manager class. It gets file in {@code indexMetadataDirectory} and tries to
+ * create a directory if it does not exist. If there already exists such a directory, it loads the
+ * serialized {@link Properties} and aggregates them within the {@link #indexMetadataMap}.
+ *
+ * @param indexMetadataDirectory
+ * the file system directory in which the index metadata will be stored
+ * @throws IndexManagementException
+ */
+ public SemanticIndexMetadataManager(File indexMetadataDirectory) throws IndexManagementException {
+ this.indexMetadataDirectory = indexMetadataDirectory;
+
+ // if directory for programs does not exist, create it
+ if (!indexMetadataDirectory.exists()) {
+ if (indexMetadataDirectory.mkdirs()) {
+ logger.info("Directory for index metadata created succesfully");
+ } else {
+ logger.error("Directory for index metadata COULD NOT be created");
+ throw new IndexManagementException("Directory : " + indexMetadataDirectory.getAbsolutePath()
+ + " cannot be created");
+ }
+ }
+
+ // load index metadata to memory
+ synchronized (indexMetadataMap) {
+ File[] metadataList = indexMetadataDirectory.listFiles();
+ for (File configFile : metadataList) {
+ String id = configFile.getName().substring(0, configFile.getName().lastIndexOf('.'));
+ Properties props = new Properties();
+ InputStream is = null;
+ try {
+ is = new FileInputStream(configFile);
+ props.load(is);
+ indexMetadataMap.put(id, props);
+ logger.info("Index metadata has been loaded from the location: {}",
+ configFile.getAbsolutePath());
+ } catch (FileNotFoundException e) {
+ logger.error("IndexMetadata file cannot be found");
+ throw new IndexManagementException("IndexMetadata file cannot be found");
+ } catch (IOException e) {
+ logger.error("Failed to read from input stream");
+ throw new IndexManagementException("Failed to read from input stream");
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns whether an index for the given {@code id} is already configured.
+ *
+ * @param id
+ * identifier of the {@link SemanticIndex}
+ * @return
+ */
+ public boolean isConfigured(String id) {
+ synchronized (indexMetadataMap) {
+ return indexMetadataMap.containsKey(id);
+ }
+ }
+
+ /**
+ * Returns the {@link Map} containing metadata of the all indexes.
+ *
+ * @return
+ */
+ public Map<String,Properties> getAllIndexMetadata() {
+ synchronized (indexMetadataMap) {
+ return indexMetadataMap;
+ }
+ }
+
+ /**
+ * Returns the metadata of the index specified with the given id
+ *
+ * @param id
+ * identifier of the {@link SemanticIndex}
+ * @return
+ */
+ public Properties getIndexMetadata(String id) {
+ synchronized (indexMetadataMap) {
+ return indexMetadataMap.get(id);
+ }
+ }
+
+ /**
+ * Stores the metadata of the index speficied with the given id either in memory and persistently.
+ *
+ * @param id
+ * identifier of the {@link SemanticIndex}
+ * @param indexMetadata
+ * @throws IndexManagementException
+ */
+ public void storeIndexMetadata(String id, Properties indexMetadata) throws IndexManagementException {
+ String indexMetadataFilePath = indexMetadataDirectory.getAbsolutePath() + File.separator + id
+ + ".props";
+ synchronized (indexMetadataMap) {
+ FileOutputStream out = null;
+ Properties stringValues = getStringValues(indexMetadata);
+ try {
+ out = new FileOutputStream(indexMetadataFilePath);
+ stringValues.store(out, null);
+ } catch (IOException e) {
+ logger.error("Failed to write indexMetadataFilePath to the specified output stream");
+ throw new IndexManagementException(
+ "Failed to write indexMetadataFilePath to the specified output stream");
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ indexMetadataMap.put(id, stringValues);
+ }
+ }
+
+ /**
+ * Updates the metadata of the index specified with the given id
+ *
+ * @param id
+ * identifier of the {@link SemanticIndex}
+ * @param indexMetadata
+ * @throws IndexManagementException
+ * @throws IOException
+ */
+ public void updateIndexMetadata(String id, Properties indexMetadata) throws IndexManagementException {
+ synchronized (indexMetadataMap) {
+ storeIndexMetadata(id, indexMetadata);
+ }
+ }
+
+ public Properties removeIndexMetadata(String id) throws IndexManagementException {
+ String indexMetadataFilePath = indexMetadataDirectory.getAbsolutePath() + File.separator + id
+ + ".props";
+ File file = new File(indexMetadataFilePath);
+ logger.info("Index metadata file to be deleted: {}", file.getAbsolutePath());
+ if (file.exists()) {
+ file.delete();
+ } else {
+ logger.error("Failed to delete IndexMetadata file");
+ throw new IndexManagementException("Failed to delete IndexMetadata file");
+ }
+
+ Properties indexMetadata;
+ synchronized (indexMetadataMap) {
+ indexMetadata = indexMetadataMap.remove(id);
+ }
+
+ return indexMetadata;
+ }
+
+ private Properties getStringValues(Properties indexMetadata) {
+ Properties properties = new Properties();
+ for (Entry<Object,Object> property : indexMetadata.entrySet()) {
+ Object value = property.getValue();
+ if (value instanceof String) {
+ properties.put(property.getKey(), value);
+ } else {
+ properties.put(property.getKey(), value.toString());
+ }
+ }
+ return properties;
+ }
+}
Added: incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/LDPathUtils.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/LDPathUtils.java?rev=1387977&view=auto
==============================================================================
--- incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/LDPathUtils.java (added)
+++ incubator/stanbol/branches/contenthub-two-layered-structure/contenthub/index/src/main/java/org/apache/stanbol/contenthub/index/solr/LDPathUtils.java Thu Sep 20 11:22:16 2012
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.contenthub.index.solr;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.compress.archivers.ArchiveException;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.ArchiveStreamFactory;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.io.IOUtils;
+import org.apache.stanbol.commons.semanticindex.index.IndexManagementException;
+import org.apache.stanbol.contenthub.servicesapi.Constants;
+import org.apache.stanbol.entityhub.ldpath.backend.SiteManagerBackend;
+import org.apache.stanbol.entityhub.servicesapi.site.SiteManager;
+import org.osgi.framework.Bundle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.bootstrap.DOMImplementationRegistry;
+import org.w3c.dom.ls.DOMImplementationLS;
+import org.w3c.dom.ls.LSOutput;
+import org.w3c.dom.ls.LSSerializer;
+import org.xml.sax.SAXException;
+
+import at.newmedialab.ldpath.api.backend.RDFBackend;
+import at.newmedialab.ldpath.model.fields.FieldMapping;
+import at.newmedialab.ldpath.model.programs.Program;
+import at.newmedialab.ldpath.parser.ParseException;
+import at.newmedialab.ldpath.parser.RdfPathParser;
+
+/**
+ * Class containing utility methods for LDPath functionalities.
+ *
+ * @author anil.sinaci
+ *
+ */
+public class LDPathUtils {
+ private static final Logger logger = LoggerFactory.getLogger(LDPathUtils.class);
+
+ public static final String NS_XSD = "http://www.w3.org/2001/XMLSchema#";
+
+ private static final String SOLR_CORE_PATH = "solr/core/";
+ private static final String SOLR_TEMPLATE_NAME = "template";
+ private static final String SOLR_TEMPLATE_ZIP = SOLR_TEMPLATE_NAME + ".zip";
+ private static final String SOLR_TEMPLATE_SCHEMA = SOLR_TEMPLATE_NAME + "/conf/schema-template.xml";
+ private static final String SOLR_SCHEMA = "/conf/schema.xml";
+ private static final String SOLR_ALLTEXT_FIELD = "contenthubreserved_text_all";
+
+ private static final int BUFFER_SIZE = 8024;
+
+ private static final Set<String> SOLR_FIELD_OPTIONS;
+ static {
+ HashSet<String> opt = new HashSet<String>();
+ opt.add("default");
+ opt.add("indexed");
+ opt.add("stored");
+ opt.add("compressed");
+ opt.add("compressThreshold");
+ opt.add("multiValued");
+ opt.add("omitNorms");
+ opt.add("omitTermFreqAndPositions");
+ opt.add("termVectors");
+ opt.add("termPositions");
+ opt.add("termOffsets");
+
+ SOLR_FIELD_OPTIONS = Collections.unmodifiableSet(opt);
+ }
+ private static final String SOLR_COPY_FIELD_OPTION = "copy";
+
+ /**
+ * A map mapping from XSD types to SOLR types.
+ */
+ private static final Map<String,String> xsdSolrTypeMap;
+ static {
+ Map<String,String> typeMap = new HashMap<String,String>();
+
+ typeMap.put(NS_XSD + "decimal", "long");
+ typeMap.put(NS_XSD + "integer", "int");
+ typeMap.put(NS_XSD + "int", "int");
+ typeMap.put(NS_XSD + "long", "long");
+ typeMap.put(NS_XSD + "short", "int");
+ typeMap.put(NS_XSD + "double", "double");
+ typeMap.put(NS_XSD + "float", "float");
+ typeMap.put(NS_XSD + "dateTime", "date");
+ typeMap.put(NS_XSD + "date", "date");
+ typeMap.put(NS_XSD + "time", "date");
+ typeMap.put(NS_XSD + "boolean", "boolean");
+ typeMap.put(NS_XSD + "anyURI", "uri");
+ typeMap.put(NS_XSD + "string", "string");
+
+ xsdSolrTypeMap = Collections.unmodifiableMap(typeMap);
+ }
+
+ private Bundle bundle;
+
+ private SiteManager siteManager;
+
+ /**
+ * Constructor taking a {@link Bundle} parameter. This bundle is used when obtaining Solr schema template.
+ *
+ * @param bundle
+ * From which the template Solr schema is obtained.
+ */
+ public LDPathUtils(Bundle bundle, SiteManager siteManager) {
+ this.bundle = bundle;
+ this.siteManager = siteManager;
+ }
+
+ /**
+ * Return the SOLR field type for the XSD type passed as argument. The xsdType needs to be a fully
+ * qualified URI. If no field type is defined, will return null.
+ *
+ * @param xsdType
+ * a URI identifying the XML Schema datatype
+ * @return
+ */
+ public String getSolrFieldType(String xsdType) {
+ String result = xsdSolrTypeMap.get(xsdType);
+ if (result == null) {
+ logger.error("Could not find SOLR field type for type " + xsdType);
+ return null;
+ } else {
+ return result;
+ }
+ }
+
+ /**
+ * Creates a {@link Reader} instance from the given program string.
+ *
+ * @param program
+ * @return a {@link InputStreamReader}.
+ * @throws IndexManagementException
+ * if {@link Constants#DEFAULT_ENCODING} is not supported
+ */
+ public static Reader constructReader(String program) throws IndexManagementException {
+ try {
+ return new InputStreamReader(new ByteArrayInputStream(
+ program.getBytes(Constants.DEFAULT_ENCODING)), Constants.DEFAULT_ENCODING);
+ } catch (UnsupportedEncodingException e) {
+ String msg = String.format("Encoding {} should be supported by the system",
+ Constants.DEFAULT_ENCODING);
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+ }
+
+ private Program<Object> getLDPathProgram(String ldPathProgram) throws IndexManagementException {
+ if (ldPathProgram == null || ldPathProgram.isEmpty()) {
+ String msg = "LDPath Program cannot be null.";
+ logger.error(msg);
+ throw new IndexManagementException(msg);
+ }
+ RDFBackend<Object> rdfBackend = new SiteManagerBackend(siteManager);
+ RdfPathParser<Object> LDparser = new RdfPathParser<Object>(rdfBackend, constructReader(ldPathProgram));
+ Program<Object> program = null;
+ try {
+ program = LDparser.parseProgram();
+ } catch (ParseException e) {
+ String msg = "Cannot parse LDPath Program";
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+ return program;
+ }
+
+ /**
+ * This method creates an {@link ArchiveInputStream} containing Solr schema configurations based on the
+ * provided <code>ldPathProgram</code>. All folders and files except <b>"schema-template.xml"</b> is took
+ * from a default Solr configuration template which is located in the resources of the bundle specified in
+ * the constructor of this class i.e {@link LDPathUtils}. Instead of the "schema-template" file, a
+ * <b>"schema.xml"</b> is created.
+ *
+ * @param coreName
+ * Name of the Solr core that is used instead of template
+ * @param ldPathProgram
+ * Program for which the Solr core will be created
+ * @return {@link ArchiveInputStream} containing the Solr configurations for the provided
+ * <code>ldPathProgram</code>
+ * @throws IndexManagementException
+ */
+ public ArchiveInputStream createSchemaArchive(String coreName, String ldPathProgram) throws IndexManagementException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream(BUFFER_SIZE);
+ ArchiveStreamFactory asf = new ArchiveStreamFactory();
+ TarArchiveOutputStream tarOutputStream = null;
+ try {
+ tarOutputStream = (TarArchiveOutputStream) asf.createArchiveOutputStream("tar", out);
+ } catch (ArchiveException e) {
+ String msg = "Cannot create an empty tar archive";
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+
+ try {
+ InputStream is = getSolrTemplateStream();
+ ZipArchiveInputStream zis = new ZipArchiveInputStream(is);
+ ZipArchiveEntry ze = null;
+ byte[] schemaFile = null;
+ while ((ze = zis.getNextZipEntry()) != null) {
+ if (SOLR_TEMPLATE_SCHEMA.equals(ze.getName())) {
+ schemaFile = createSchemaXML(getLDPathProgram(ldPathProgram), IOUtils.toByteArray(zis));
+ TarArchiveEntry te = new TarArchiveEntry(coreName + SOLR_SCHEMA);
+ te.setSize(schemaFile.length);
+ tarOutputStream.putArchiveEntry(te);
+ tarOutputStream.write(schemaFile);
+ tarOutputStream.closeArchiveEntry();
+ } else {
+ TarArchiveEntry te = new TarArchiveEntry(ze.getName().replaceAll(SOLR_TEMPLATE_NAME,
+ coreName));
+ te.setSize(ze.getSize());
+ tarOutputStream.putArchiveEntry(te);
+ tarOutputStream.write(IOUtils.toByteArray(zis));
+ tarOutputStream.closeArchiveEntry();
+ }
+
+ }
+ if (schemaFile == null) {
+ throw new IndexManagementException("Schema template ZIP should include: "
+ + SOLR_TEMPLATE_SCHEMA);
+ }
+ tarOutputStream.finish();
+ tarOutputStream.close();
+ } catch (IOException e) {
+ logger.error("", e);
+ throw new IndexManagementException(e);
+ }
+
+ ArchiveInputStream ret;
+ try {
+ ret = asf.createArchiveInputStream(new ByteArrayInputStream(out.toByteArray()));
+ } catch (ArchiveException e) {
+ String msg = "Cannot create a final tar archive while creating an ArchiveInputStream to create a Solr core";
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+ return ret;
+ }
+
+ private InputStream getSolrTemplateStream() throws IndexManagementException {
+ String solrCorePath = SOLR_CORE_PATH;
+ if (!solrCorePath.endsWith(File.separator)) solrCorePath += File.separator;
+ String templateZip = solrCorePath + SOLR_TEMPLATE_ZIP;
+
+ URL resource = bundle.getEntry(templateZip);
+ InputStream is = null;
+ try {
+ is = resource != null ? resource.openStream() : null;
+ } catch (IOException e) {
+ String msg = "Cannot open input stream on URL resource gathered from bundle.getEntry()";
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+ if (is == null) {
+ String msg = "Solr Template ZIP cannot be found in:" + templateZip;
+ logger.error(msg);
+ throw new IndexManagementException(msg);
+ }
+ return is;
+ }
+
+ /**
+ * Creates <b>"schema.xml"</b> file for the Solr configurations to be created for the provided LDPath
+ * program. Creates <b>Solr fields</b> for each field obtained by calling {@link Program#getFields()} of
+ * provided <code>program</code>. By default, <i>name</i>, <i>type</i>, <i>stored</i>, <i>indexed</i> and
+ * <i>multiValued</i> attributes of fields are set. Furthermore, any attribute obtained from the fields of
+ * the program is also set if it is included in {@link LDPathUtils#SOLR_FIELD_OPTIONS}. Another
+ * configuration about the fields obtained from the program is {@link LDPathUtils#SOLR_COPY_FIELD_OPTION}.
+ * If there is a specified configuration about this field, <b>destination</b> of <b>copyField</b> element
+ * is set accordingly. Otherwise, the destination is set as <b>contenthubreserved_text_all</b>
+ *
+ * @param program
+ * LDPath program of which fields will be obtained
+ * @param template
+ * Solr schema template to be populated with the fields based on the provided
+ * <code>program</code>
+ * @return created template in an array of bytes.
+ * @throws IndexManagementException
+ */
+ private byte[] createSchemaXML(Program<Object> program, byte[] template) throws IndexManagementException {
+ Document document;
+ try {
+ document = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+ .parse(new ByteArrayInputStream(template));
+ } catch (SAXException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ } catch (IOException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ } catch (ParserConfigurationException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+
+ Element schemaNode = document.getDocumentElement();
+ NodeList fieldsNodeList = schemaNode.getElementsByTagName("fields");
+ if (fieldsNodeList.getLength() != 1) {
+ throw new IndexManagementException(
+ "Template is an invalid SOLR schema. It should be a valid a byte array");
+ }
+
+ Node fieldsNode = fieldsNodeList.item(0);
+
+ for (FieldMapping<?,Object> fieldMapping : program.getFields()) {
+ String fieldName = fieldMapping.getFieldName();
+ String solrType = getSolrFieldType(fieldMapping.getFieldType());
+
+ if (solrType == null) {
+ logger.error("field {} has an invalid field type; ignoring field definition", fieldName);
+ } else {
+ Element fieldElement = document.createElement("field");
+ fieldElement.setAttribute("name", fieldName);
+ fieldElement.setAttribute("type", solrType);
+ fieldElement.setAttribute("stored", "true");
+ fieldElement.setAttribute("indexed", "true");
+ fieldElement.setAttribute("multiValued", "true");
+
+ // Handle extra field configuration
+ final Map<String,String> fieldConfig = fieldMapping.getFieldConfig();
+ if (fieldConfig != null) {
+ for (String attr : fieldConfig.keySet()) {
+ if (SOLR_FIELD_OPTIONS.contains(attr)) {
+ fieldElement.setAttribute(attr, fieldConfig.get(attr));
+ }
+ }
+ }
+ fieldsNode.appendChild(fieldElement);
+
+ if (fieldConfig != null && fieldConfig.keySet().contains(SOLR_COPY_FIELD_OPTION)) {
+ String[] copyFields = fieldConfig.get(SOLR_COPY_FIELD_OPTION).split(",\\s*");
+ for (String copyField : copyFields) {
+ Element copyElement = document.createElement("copyField");
+ copyElement.setAttribute("source", fieldName);
+ copyElement.setAttribute("dest", copyField);
+ schemaNode.appendChild(copyElement);
+ }
+ } else {
+ Element copyElement = document.createElement("copyField");
+ copyElement.setAttribute("source", fieldName);
+ copyElement.setAttribute("dest", SOLR_ALLTEXT_FIELD);
+ schemaNode.appendChild(copyElement);
+ }
+ }
+ }
+
+ DOMImplementationRegistry registry;
+ try {
+ registry = DOMImplementationRegistry.newInstance();
+ } catch (ClassCastException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ } catch (ClassNotFoundException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ } catch (InstantiationException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ } catch (IllegalAccessException e) {
+ String msg = e.getMessage();
+ logger.error(msg, e);
+ throw new IndexManagementException(msg, e);
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DOMImplementationLS lsImpl = (DOMImplementationLS) registry.getDOMImplementation("LS");
+ LSSerializer lsSerializer = lsImpl.createLSSerializer();
+ LSOutput lsOutput = lsImpl.createLSOutput();
+ lsOutput.setEncoding("UTF-8");
+ lsOutput.setByteStream(baos);
+ lsSerializer.write(document, lsOutput);
+ String schemaStr = new String(baos.toByteArray());
+ return schemaStr.getBytes();
+ }
+
+}