You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/08/14 18:35:29 UTC
svn commit: r1157558 - in /incubator/opennlp/sandbox/corpus-server/src/main:
java/org/apache/opennlp/corpus_server/
java/org/apache/opennlp/corpus_server/search/
java/org/apache/opennlp/corpus_server/store/
resources/org/apache/opennlp/corpus_server/se...
Author: joern
Date: Sun Aug 14 16:35:27 2011
New Revision: 1157558
URL: http://svn.apache.org/viewvc?rev=1157558&view=rev
Log:
OPENNLP-207 Improved lucas integration
Added:
incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties (with props)
incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml (with props)
incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml (contents, props changed)
- copied, changed from r1155976, incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml
Removed:
incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml
Modified:
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java
incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java Sun Aug 14 16:35:27 2011
@@ -58,6 +58,6 @@ public class CorporaResource {
CorpusServer corpusServer = CorpusServer.getInstance();
CorporaStore store = corpusServer.getStore();
- return new CorpusResource(store.getCorpus(corpus));
+ return new CorpusResource(store.getCorpus(corpus), corpusServer.getSearchService());
}
}
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java Sun Aug 14 16:35:27 2011
@@ -20,6 +20,8 @@ package org.apache.opennlp.corpus_server
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
@@ -31,17 +33,20 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
+import org.apache.opennlp.corpus_server.search.SearchService;
import org.apache.opennlp.corpus_server.store.CorpusStore;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.xml.sax.SAXException;
public class CorpusResource {
- private final CorpusStore corpus;
+ private final CorpusStore corpus;
+ private final SearchService service;
- public CorpusResource(CorpusStore corpus) {
- this.corpus = corpus;
- }
+ public CorpusResource(CorpusStore corpus, SearchService service) {
+ this.corpus = corpus;
+ this.service = service;
+ }
/**
* Adds a new CAS to the store.
@@ -107,6 +112,6 @@ public class CorpusResource {
@Produces(MediaType.APPLICATION_JSON)
@Path("_search")
public Collection<String> search(@QueryParam("q") String q) throws IOException {
- return null; // corpus.search(q);
+ return service.search(corpus, q);
}
}
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java Sun Aug 14 16:35:27 2011
@@ -82,7 +82,7 @@ public class CorpusServer implements Ser
searchService = new LuceneSearchService();
try {
- searchService.initialize();
+ searchService.initialize(store);
} catch (IOException e) {
LOGGER.log(Level.SEVERE, "Failed to start search service!", e);
}
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java Sun Aug 14 16:35:27 2011
@@ -17,23 +17,38 @@
package org.apache.opennlp.corpus_server.search;
-import java.io.File;
import java.io.IOException;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.NIOFSDirectory;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
import org.apache.uima.lucas.LuceneDocumentAE;
+import org.apache.uima.lucas.indexer.IndexWriterProvider;
+import org.apache.uima.resource.ResourceAccessException;
import org.apache.uima.resource.ResourceInitializationException;
public class LuceneIndexer extends LuceneDocumentAE {
+ static final String CAS_ID_TYPE = "org.apache.opennlp.corpus_server.CasId";
+ static final String CAS_ID_FEEATURE = "id";
+
+ private static final String RESOURCE_INDEX_WRITER_PROVIDER = "indexWriterProvider";
+
+ private Type casIdType;
+ private Feature casIdFeature;
+
private IndexWriter indexWriter;
@Override
@@ -41,33 +56,52 @@ public class LuceneIndexer extends Lucen
throws ResourceInitializationException {
super.initialize(aContext);
- Directory indexDir = null;
- try {
- indexDir = new NIOFSDirectory(new File("test-index"));
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
+ IndexWriterProvider indexWriterProvider;
try {
- indexWriter = new IndexWriter(indexDir, null, true);
- } catch (CorruptIndexException e) {
- throw new ResourceInitializationException(e);
- } catch (LockObtainFailedException e) {
- throw new ResourceInitializationException(e);
- } catch (IOException e) {
+ indexWriterProvider = (IndexWriterProvider) aContext
+ .getResourceObject(RESOURCE_INDEX_WRITER_PROVIDER);
+ } catch (ResourceAccessException e) {
throw new ResourceInitializationException(e);
}
+
+ indexWriter = indexWriterProvider.getIndexWriter();
}
+ @Override
+ public void typeSystemInit(TypeSystem aTypeSystem)
+ throws AnalysisEngineProcessException {
+ super.typeSystemInit(aTypeSystem);
+
+ casIdType = aTypeSystem.getType(CAS_ID_TYPE);
+ casIdFeature = casIdType.getFeatureByBaseName(CAS_ID_FEEATURE);
+ }
@Override
public void process(CAS cas) throws AnalysisEngineProcessException {
-
+
+ String casId = null;
+
+ for (FSIterator<FeatureStructure> casIdIterator =
+ cas.getIndexRepository().getAllIndexedFS(casIdType); casIdIterator.hasNext();) {
+ FeatureStructure casIdFS = casIdIterator.next();
+
+ casId = casIdFS.getStringValue(casIdFeature);
+ }
+
+ if (casId == null)
+ throw new AnalysisEngineProcessException(new Exception("Missing cas id feature structure!"));
+
+ Query idQuery = new TermQuery(new Term("id", casId));
+
Document doc = createDocument(cas);
-
+ doc.add(new Field("id", casId, Field.Store.YES, Field.Index.NOT_ANALYZED));
+
try {
+ indexWriter.deleteDocuments(idQuery);
indexWriter.addDocument(doc);
+
+ // TODO: Commit handling might need to be changed
+ indexWriter.commit();
} catch (CorruptIndexException e) {
throw new AnalysisEngineProcessException(e);
} catch (IOException e) {
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java Sun Aug 14 16:35:27 2011
@@ -19,22 +19,46 @@ package org.apache.opennlp.corpus_server
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
import org.apache.opennlp.corpus_server.UimaUtil;
+import org.apache.opennlp.corpus_server.store.CorporaStore;
import org.apache.opennlp.corpus_server.store.CorpusStore;
import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.resource.FileResourceSpecifier;
import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.impl.FileResourceSpecifier_impl;
+import org.apache.uima.resource.metadata.MetaDataObject;
+import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.util.CasCreationUtils;
import org.apache.uima.util.InvalidXMLException;
import org.apache.uima.util.XMLInputSource;
@@ -44,53 +68,160 @@ public class LuceneSearchService impleme
private final static Logger LOGGER = Logger.getLogger(
LuceneSearchService.class .getName());
- private AnalysisEngine indexer;
- AnalysisEngineDescription specifier;
+ private Map<String, AnalysisEngine> corpusIndexerMap = new HashMap<String, AnalysisEngine>();
// create a map with corpus name and indexer ae ...
// indexer ae is a pair of ae and descriptor (maybe that can be done nicer)
+
+ private static File getIndexDirectory(String corpusId) {
+ return new File("index" + File.separator + corpusId);
+ }
@Override
- public void initialize() throws IOException {
-
- // TODO: We need an indexer per corpus ...
- // For each corpus ...
+ public synchronized void initialize(CorporaStore corporaStore) throws IOException {
+ // TODO: Need to take care for thread safety ..
- //get Resource Specifier from XML file
- XMLInputSource in = new XMLInputSource(LuceneSearchService.class.getResourceAsStream(
- "/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml"), new File(""));
-
- try {
- specifier = (AnalysisEngineDescription) UIMAFramework.getXMLParser().parseResourceSpecifier(in);
+ for (String corpusId : corporaStore.getCorpusIds()) {
- // TODO: Place mapping in a tmp file, and then set the path to it ...
- // How to modify this parameter?!
- // TODO: Specify the storage location for the index ...
- indexer = UIMAFramework.produceAnalysisEngine(specifier); // NOTE: This will use the wrong type system ...
- } catch (InvalidXMLException e) {
- throw new IOException(e);
- } catch (ResourceInitializationException e) {
- throw new IOException(e);
+ // Set the index mapping file for this corpus in the analysis engine descriptor
+
+ XMLInputSource in = new XMLInputSource(LuceneSearchService.class.getResourceAsStream(
+ "/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml"), new File(""));
+
+ try {
+ AnalysisEngineDescription specifier;
+ specifier = (AnalysisEngineDescription) UIMAFramework.getXMLParser().parseResourceSpecifier(in);
+
+ // TODO: How to store mapping file? Should be transmitted during corpus creation ...
+
+ File mappingTmpFile = File.createTempFile("lucas-mapping", corpusId + ".xml");
+ mappingTmpFile.deleteOnExit();
+
+ InputStream mappingFileIn = null;
+ OutputStream mappingTmpOut = null;
+
+ try {
+ mappingFileIn = LuceneSearchService.class.getResourceAsStream(
+ "/org/apache/opennlp/corpus_server/search/" + corpusId + ".xml");
+ mappingTmpOut = new FileOutputStream(mappingTmpFile);
+
+ byte buffer[] = new byte[1024];
+ int len = 0;
+ while ((len = mappingFileIn.read(buffer)) > 0) {
+ mappingTmpOut.write(buffer, 0, len);
+ }
+ }
+ catch (IOException e) {
+ // TODO: Or just ignore it ?! and do not create the indexer for this corpus?!
+ throw e;
+ }
+ finally {
+ if (mappingFileIn != null) {
+ try {
+ mappingFileIn.close();
+ }
+ catch (IOException e) {}
+ }
+
+ if (mappingTmpOut != null) {
+ try {
+ mappingTmpOut.close();
+ }
+ catch (IOException e) {}
+ }
+ }
+
+
+ specifier.getAnalysisEngineMetaData().
+ getConfigurationParameterSettings().setParameterValue("mappingFile",
+ mappingTmpFile.getAbsolutePath());
+
+ // Set the index writer properties file in the analysis engine
+ // and replace the index path with the index location for this corpus
+
+ Properties indexWriterProperties = new Properties();
+
+ InputStream indexWriterPropertiesIn = null;
+ try {
+ // TODO: Retrieve file form somewhere for this corpus
+ indexWriterPropertiesIn = LuceneSearchService.class.getResourceAsStream(
+ "/org/apache/opennlp/corpus_server/search/IndexWriter.properties");
+
+ indexWriterProperties.load(indexWriterPropertiesIn);
+ }
+ finally {
+ if (indexWriterPropertiesIn != null) {
+ try {
+ indexWriterPropertiesIn.close();
+ }
+ catch (IOException e) {}
+ }
+ }
+
+ indexWriterProperties.setProperty("indexPath", getIndexDirectory(corpusId).getAbsolutePath());
+
+ File indexWriterTmpFile = File.createTempFile("index-writer", corpusId + ".properties");
+ indexWriterTmpFile.deleteOnExit();
+
+ OutputStream indexPropertiesOut = null;
+ try {
+ indexPropertiesOut = new FileOutputStream(indexWriterTmpFile);
+ // write properties into a tmp file
+ indexWriterProperties.store(indexPropertiesOut, null);
+ }
+ finally {
+ if (indexPropertiesOut != null) {
+ try {
+ indexPropertiesOut.close();
+ }
+ catch (IOException e) {}
+ }
+ }
+
+ FileResourceSpecifier indexWriterFileSpecifier = new FileResourceSpecifier_impl();
+ indexWriterFileSpecifier.setFileUrl(indexWriterTmpFile.toURL().toString());
+ // TODO: This will fail ...
+ specifier.getResourceManagerConfiguration().getExternalResources()[0].setResourceSpecifier(indexWriterFileSpecifier);
+
+ AnalysisEngine indexer = UIMAFramework.produceAnalysisEngine(specifier);
+ corpusIndexerMap.put(corpusId, indexer);
+ } catch (InvalidXMLException e) {
+ throw new IOException(e);
+ } catch (ResourceInitializationException e) {
+ throw new IOException(e);
+ }
}
}
@Override
- public void index(CorpusStore store, String casId) throws IOException {
-
- // Get a reference to the indexer for this corpus ...
+ public synchronized void index(CorpusStore store, String casId) throws IOException {
// TODO: Need to take care for thread safety ..
- // TODO: AE type system must be merged with corpus type system ..
- // CasCreationUtils.createCas(list);
- // Does that also work with a type system description?!
- List specs = new ArrayList();
- specs.add(indexer); // Note: Is this allowed?
+ String corpusId = store.getCorpusId();
+
+ AnalysisEngine indexer = corpusIndexerMap.get(corpusId);
+
+
+ InputStream indexTsIn = LuceneSearchService.class.getResourceAsStream(
+ "/org/apache/opennlp/corpus_server/search/TypeSystem.xml");
+
+ TypeSystemDescription indexTypeDesc;
+ try {
+ indexTypeDesc = UimaUtil.createTypeSystemDescription(indexTsIn);
+ }
+ finally {
+ indexTsIn.close();
+ }
+
+ List<MetaDataObject> specs = new ArrayList<MetaDataObject>();
+ specs.add(indexTypeDesc);
specs.add(store.getTypeSystem());
+ // Note: This might be a performance problem
CAS cas;
try {
cas = CasCreationUtils.createCas(specs);
@@ -102,6 +233,14 @@ public class LuceneSearchService impleme
UimaUtil.deserializeXmiCAS(cas, new ByteArrayInputStream(casBytes));
+ // Inject id feature structure into the CAS
+ Type casIdType = cas.getTypeSystem().getType(LuceneIndexer.CAS_ID_TYPE);
+ Feature casIdFeature = casIdType.getFeatureByBaseName(LuceneIndexer.CAS_ID_FEEATURE);
+
+ FeatureStructure casIdFS = cas.createFS(casIdType);
+ casIdFS.setStringValue(casIdFeature, casId);
+ cas.addFsToIndexes(casIdFS);
+
try {
indexer.process(cas);
} catch (AnalysisEngineProcessException e) {
@@ -115,15 +254,64 @@ public class LuceneSearchService impleme
public Collection<String> search(CorpusStore store, String q)
throws IOException {
+ // TODO:
+ // Creating a reader per query is can only be done for testing this ...
+ // Open/Reopen index once in a while to see updates, can this be done automatically?!
+
+ File indexLocation = getIndexDirectory(store.getCorpusId());
+
+ QueryParser parser = null;
+
+ final IndexSearcher searcher = new IndexSearcher(indexLocation.getAbsolutePath());
+ parser = new QueryParser("content", new StandardAnalyzer());
+
+ Query query;
+ try {
+ query = parser.parse(q);
+ } catch (ParseException e) {
+ throw new IOException(e);
+ }
+
+ final Set<String> results = new HashSet<String>();
+
// query index ...
+ searcher.search(query, new Collector() {
+
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ }
+
+ @Override
+ public void setNextReader(IndexReader reader, int arg1) throws IOException {
+ }
+
+ @Override
+ public void collect(int id) throws IOException {
+ Document doc = searcher.doc(id);
+ String idString = doc.get("id");
+ results.add(idString);
+ }
+
+ @Override
+ public boolean acceptsDocsOutOfOrder() {
+ return false;
+ }
+ });
+
+ searcher.close();
- return Collections.emptySet();
+ return results;
}
@Override
public void shutdown() throws IOException {
- if (indexer != null) {
- indexer.destroy();
+
+ for (String corpusId : corpusIndexerMap.keySet()) {
+ AnalysisEngine indexer = corpusIndexerMap.get(corpusId);
+
+ if (indexer != null) {
+ indexer.destroy();
+ }
}
}
}
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java Sun Aug 14 16:35:27 2011
@@ -20,11 +20,12 @@ package org.apache.opennlp.corpus_server
import java.io.IOException;
import java.util.Collection;
+import org.apache.opennlp.corpus_server.store.CorporaStore;
import org.apache.opennlp.corpus_server.store.CorpusStore;
public interface SearchService {
- void initialize() throws IOException;
+ void initialize(CorporaStore corporaStore) throws IOException;
// index
void index(CorpusStore store, String casId) throws IOException;
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java Sun Aug 14 16:35:27 2011
@@ -18,6 +18,7 @@
package org.apache.opennlp.corpus_server.store;
import java.io.IOException;
+import java.util.Set;
/**
* The Corpora Store is the central storage which manages the corpora.
@@ -45,6 +46,8 @@ public interface CorporaStore {
*/
void createCorpus(String corpusName, byte typeSystemBytes[]) throws IOException;
+ Set<String> getCorpusIds() throws IOException;
+
/**
* Retrieves a corpus of the given name from the store.
*
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java Sun Aug 14 16:35:27 2011
@@ -27,6 +27,8 @@ import org.apache.uima.resource.metadata
*/
public interface CorpusStore {
+ String getCorpusId();
+
byte[] getCAS(String casId) throws IOException;
void addCAS(String casID, byte[] content) throws IOException;
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java Sun Aug 14 16:35:27 2011
@@ -20,10 +20,15 @@ package org.apache.opennlp.corpus_server
import java.io.IOException;
import java.sql.Blob;
import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -86,6 +91,37 @@ public class DerbyCorporaStore extends A
}
}
+ public Set<String> getCorpusIds() throws IOException {
+
+ Set<String> corpusIds = new HashSet<String>();
+
+ try {
+ Connection conn = dataSource.getConnection();
+ DatabaseMetaData dbmd = conn.getMetaData();
+
+ String[] types = { "TABLE" };
+ ResultSet resultSet = dbmd.getTables(null, null, "%", types);
+
+ while (resultSet.next()) {
+ String tableName = resultSet.getString(3);
+ corpusIds.add(tableName.toLowerCase());
+ }
+
+ conn.close();
+
+ } catch (SQLException e) {
+
+ if (LOGGER.isLoggable(Level.SEVERE)) {
+ LOGGER.log(Level.SEVERE, "Failed to retrieve corpus ids!", e);
+ }
+
+ throw new IOException(e);
+ }
+
+
+ return Collections.unmodifiableSet(corpusIds);
+ }
+
@Override
public CorpusStore getCorpus(String corpusId) {
return new DerbyCorpusStore(dataSource, this, corpusId);
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java Sun Aug 14 16:35:27 2011
@@ -40,7 +40,7 @@ public class DerbyCorpusStore implements
private DataSource dataSource;
private DerbyCorporaStore store;
- private String corpusName;
+ private final String corpusName;
DerbyCorpusStore(DataSource dataSource, DerbyCorporaStore store, String corpusName) {
this.dataSource = dataSource;
@@ -49,6 +49,11 @@ public class DerbyCorpusStore implements
}
@Override
+ public String getCorpusId() {
+ return corpusName;
+ }
+
+ @Override
public byte[] getCAS(String casId) throws IOException {
byte casBytes[] = null;
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java Sun Aug 14 16:35:27 2011
@@ -18,8 +18,11 @@
package org.apache.opennlp.corpus_server.store;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.opennlp.corpus_server.UimaUtil;
@@ -44,6 +47,11 @@ public class MemoryCorporaStore extends
return instance;
}
+ @Override
+ public Set<String> getCorpusIds() throws IOException {
+ return Collections.unmodifiableSet(corpora.keySet());
+ }
+
// Note: Add one twice, overwrites an existing one!
public void createCorpus(String corpusName,
byte typeSystemBytes[]) {
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java Sun Aug 14 16:35:27 2011
@@ -41,6 +41,11 @@ public class MemoryCorpusStore implement
this.typeSystem = typeSystem;
}
+ @Override
+ public String getCorpusId() {
+ return corpusName;
+ }
+
public byte[] getCAS(String casId) {
return casStore.get(casId);
}
Added: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties?rev=1157558&view=auto
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties (added)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties Sun Aug 14 16:35:27 2011
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The indexPath replaced by the analysis engine integration
+# indexPath =
+#RAMBufferSize = 513
+#useCompoundFileFormat = false
+#maxFieldLength = 9999
+#uniqueIndex = true
\ No newline at end of file
Propchange: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml Sun Aug 14 16:35:27 2011
@@ -41,7 +41,7 @@
<nameValuePair>
<name>mappingFile</name>
<value>
- <string>/Users/joern/dev/opennlp-apache/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml</string>
+ <string>file-will-be-set-by-ae-integration</string>
</value>
</nameValuePair>
</configurationParameterSettings>
@@ -61,4 +61,24 @@
<outputsNewCASes>false</outputsNewCASes>
</operationalProperties>
</processingResourceMetaData>
+
+ <resourceManagerConfiguration>
+ <externalResources>
+ <externalResource>
+ <name>LucasIndexWriterProvider</name>
+ <description>Provides a index writer instance.</description>
+ <fileResourceSpecifier>
+ <fileUrl>file:src/test/resources/IndexWriter.properties</fileUrl>
+ </fileResourceSpecifier>
+ <implementationName>org.apache.uima.lucas.indexer.IndexWriterProviderImpl</implementationName>
+ </externalResource>
+ </externalResources>
+ <externalResourceBindings>
+ <externalResourceBinding>
+ <key>indexWriterProvider</key>
+ <resourceName>LucasIndexWriterProvider</resourceName>
+ </externalResourceBinding>
+ </externalResourceBindings>
+ </resourceManagerConfiguration>
+
</analysisEngineDescription>
Added: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml?rev=1157558&view=auto
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml (added)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml Sun Aug 14 16:35:27 2011
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+ <!--
+ ***************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ ***************************************************************
+ -->
+
+<typeSystemDescription xmlns="http://uima.apache.org/resourceSpecifier">
+ <name>IndexerTypeSystem</name>
+ <types>
+ <typeDescription>
+ <name>org.apache.opennlp.corpus_server.CasId</name>
+ <supertypeName>uima.cas.TOP</supertypeName>
+ <features>
+ <featureDescription>
+ <name>id</name>
+ <description></description>
+ <rangeTypeName>uima.cas.String</rangeTypeName>
+ </featureDescription>
+ </features>
+ </typeDescription>
+ </types>
+</typeSystemDescription>
\ No newline at end of file
Propchange: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Copied: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml (from r1155976, incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml)
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml?p2=incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml&p1=incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml&r1=1155976&r2=1157558&rev=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml Sun Aug 14 16:35:27 2011
@@ -22,7 +22,7 @@
<fields xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="../../main/resources/lucas.xsd">
- <field name="headline" index="yes" stored="yes">
+ <field name="headline" index="yes" stored="no">
<annotations>
<annotation type="org.apache.opennlp.annotations.Headline" tokenizer="standard"/>
</annotations>
Propchange: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain