You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/08/14 18:35:29 UTC

svn commit: r1157558 - in /incubator/opennlp/sandbox/corpus-server/src/main: java/org/apache/opennlp/corpus_server/ java/org/apache/opennlp/corpus_server/search/ java/org/apache/opennlp/corpus_server/store/ resources/org/apache/opennlp/corpus_server/se...

Author: joern
Date: Sun Aug 14 16:35:27 2011
New Revision: 1157558

URL: http://svn.apache.org/viewvc?rev=1157558&view=rev
Log:
OPENNLP-207 Improved lucas integration

Added:
    incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties   (with props)
    incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml   (with props)
    incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml   (contents, props changed)
      - copied, changed from r1155976, incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml
Removed:
    incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml
Modified:
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java
    incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java
    incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java Sun Aug 14 16:35:27 2011
@@ -58,6 +58,6 @@ public class CorporaResource {
       CorpusServer corpusServer = CorpusServer.getInstance();
       CorporaStore store = corpusServer.getStore();
     
-      return new CorpusResource(store.getCorpus(corpus));
+      return new CorpusResource(store.getCorpus(corpus), corpusServer.getSearchService());
 	}
 }

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusResource.java Sun Aug 14 16:35:27 2011
@@ -20,6 +20,8 @@ package org.apache.opennlp.corpus_server
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 
 import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
@@ -31,17 +33,20 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 
+import org.apache.opennlp.corpus_server.search.SearchService;
 import org.apache.opennlp.corpus_server.store.CorpusStore;
 import org.apache.uima.resource.metadata.TypeSystemDescription;
 import org.xml.sax.SAXException;
 
 public class CorpusResource {
 
-	private final CorpusStore corpus;
+  private final CorpusStore corpus;
+  private final SearchService service;
 	
-	public CorpusResource(CorpusStore corpus) {
-		this.corpus = corpus;
-	}
+  public CorpusResource(CorpusStore corpus, SearchService service) {
+    this.corpus = corpus;
+    this.service = service;
+  }
 	
 	/**
 	 * Adds a new CAS to the store.
@@ -107,6 +112,6 @@ public class CorpusResource {
 	@Produces(MediaType.APPLICATION_JSON)
 	@Path("_search")
 	public Collection<String> search(@QueryParam("q") String q) throws IOException {
-		return null; // corpus.search(q);
+	  return service.search(corpus, q);
 	}
 }

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorpusServer.java Sun Aug 14 16:35:27 2011
@@ -82,7 +82,7 @@ public class CorpusServer implements Ser
     searchService = new LuceneSearchService();
     
     try {
-      searchService.initialize();
+      searchService.initialize(store);
     } catch (IOException e) {
       LOGGER.log(Level.SEVERE, "Failed to start search service!", e);
     }

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneIndexer.java Sun Aug 14 16:35:27 2011
@@ -17,23 +17,38 @@
 
 package org.apache.opennlp.corpus_server.search;
 
-import java.io.File;
 import java.io.IOException;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.NIOFSDirectory;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.uima.UimaContext;
 import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
 import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
 import org.apache.uima.lucas.LuceneDocumentAE;
+import org.apache.uima.lucas.indexer.IndexWriterProvider;
+import org.apache.uima.resource.ResourceAccessException;
 import org.apache.uima.resource.ResourceInitializationException;
 
 public class LuceneIndexer extends LuceneDocumentAE {
 
+  static final String CAS_ID_TYPE = "org.apache.opennlp.corpus_server.CasId";
+  static final String CAS_ID_FEEATURE = "id";
+  
+  private static final String RESOURCE_INDEX_WRITER_PROVIDER = "indexWriterProvider";
+  
+  private Type casIdType;
+  private Feature casIdFeature;
+  
   private IndexWriter indexWriter;
   
   @Override
@@ -41,33 +56,52 @@ public class LuceneIndexer extends Lucen
       throws ResourceInitializationException {
     super.initialize(aContext);
     
-    Directory indexDir = null;
-    try {
-      indexDir = new NIOFSDirectory(new File("test-index"));
-    } catch (IOException e) {
-      // TODO Auto-generated catch block
-      e.printStackTrace();
-    }
-    
+    IndexWriterProvider indexWriterProvider;
     try {
-      indexWriter = new IndexWriter(indexDir, null, true);
-    } catch (CorruptIndexException e) {
-      throw new ResourceInitializationException(e);
-    } catch (LockObtainFailedException e) {
-      throw new ResourceInitializationException(e);
-    } catch (IOException e) {
+      indexWriterProvider = (IndexWriterProvider) aContext
+          .getResourceObject(RESOURCE_INDEX_WRITER_PROVIDER);
+    } catch (ResourceAccessException e) {
       throw new ResourceInitializationException(e);
     }
+    
+    indexWriter = indexWriterProvider.getIndexWriter();
   }
   
+  @Override
+  public void typeSystemInit(TypeSystem aTypeSystem)
+      throws AnalysisEngineProcessException {
+    super.typeSystemInit(aTypeSystem);
+    
+    casIdType = aTypeSystem.getType(CAS_ID_TYPE);
+    casIdFeature = casIdType.getFeatureByBaseName(CAS_ID_FEEATURE);
+  }
   
   @Override
   public void process(CAS cas) throws AnalysisEngineProcessException {
-
+    
+    String casId = null;
+    
+    for (FSIterator<FeatureStructure> casIdIterator = 
+        cas.getIndexRepository().getAllIndexedFS(casIdType); casIdIterator.hasNext();) {
+      FeatureStructure casIdFS = casIdIterator.next();
+      
+      casId = casIdFS.getStringValue(casIdFeature);
+    }
+    
+    if (casId == null)
+      throw new AnalysisEngineProcessException(new Exception("Missing cas id feature structure!"));
+    
+    Query idQuery = new TermQuery(new Term("id", casId));
+      
     Document doc = createDocument(cas);
-  
+    doc.add(new Field("id", casId, Field.Store.YES, Field.Index.NOT_ANALYZED));
+    
     try {
+      indexWriter.deleteDocuments(idQuery);
       indexWriter.addDocument(doc);
+      
+      // TODO: Commit handling might need to be changed
+      indexWriter.commit();
     } catch (CorruptIndexException e) {
       throw new AnalysisEngineProcessException(e);
     } catch (IOException e) {

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java Sun Aug 14 16:35:27 2011
@@ -19,22 +19,46 @@ package org.apache.opennlp.corpus_server
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
 import org.apache.opennlp.corpus_server.UimaUtil;
+import org.apache.opennlp.corpus_server.store.CorporaStore;
 import org.apache.opennlp.corpus_server.store.CorpusStore;
 import org.apache.uima.UIMAFramework;
 import org.apache.uima.analysis_engine.AnalysisEngine;
 import org.apache.uima.analysis_engine.AnalysisEngineDescription;
 import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
 import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.resource.FileResourceSpecifier;
 import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.impl.FileResourceSpecifier_impl;
+import org.apache.uima.resource.metadata.MetaDataObject;
+import org.apache.uima.resource.metadata.TypeSystemDescription;
 import org.apache.uima.util.CasCreationUtils;
 import org.apache.uima.util.InvalidXMLException;
 import org.apache.uima.util.XMLInputSource;
@@ -44,53 +68,160 @@ public class LuceneSearchService impleme
   private final static Logger LOGGER = Logger.getLogger(
       LuceneSearchService.class .getName());
   
-  private AnalysisEngine indexer;
-  AnalysisEngineDescription specifier;
+  private Map<String, AnalysisEngine> corpusIndexerMap = new HashMap<String, AnalysisEngine>();
   
   // create a map with corpus name and indexer ae ...
   // indexer ae is a pair of ae and descriptor (maybe that can be done nicer)
+
+  private static File getIndexDirectory(String corpusId) {
+    return new File("index" + File.separator + corpusId);
+  }
   
   @Override
-  public void initialize() throws IOException {
-    
-    // TODO: We need an indexer per corpus ... 
-    // For each corpus ...
+  public synchronized void initialize(CorporaStore corporaStore) throws IOException {
     
+ // TODO: Need to take care for thread safety ..
     
-    //get Resource Specifier from XML file
-    XMLInputSource in = new XMLInputSource(LuceneSearchService.class.getResourceAsStream(
-        "/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml"), new File(""));
-    
-    try {
-      specifier = (AnalysisEngineDescription) UIMAFramework.getXMLParser().parseResourceSpecifier(in);
+    for (String corpusId : corporaStore.getCorpusIds()) {
       
-      // TODO: Place mapping in a tmp file, and then set the path to it ...
-      // How to modify this parameter?!
-      // TODO: Specify the storage location for the index ...
       
-      indexer = UIMAFramework.produceAnalysisEngine(specifier); // NOTE: This will use the wrong type system ... 
-    } catch (InvalidXMLException e) {
-      throw new IOException(e);
-    } catch (ResourceInitializationException e) {
-      throw new IOException(e);
+      // Set the index mapping file for this corpus in the analysis engine descriptor
+      
+      XMLInputSource in = new XMLInputSource(LuceneSearchService.class.getResourceAsStream(
+          "/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml"), new File(""));
+      
+      try {
+        AnalysisEngineDescription specifier;
+        specifier = (AnalysisEngineDescription) UIMAFramework.getXMLParser().parseResourceSpecifier(in);
+        
+        // TODO: How to store mapping file? Should be transmitted during corpus creation ...
+
+        File mappingTmpFile = File.createTempFile("lucas-mapping", corpusId + ".xml");
+        mappingTmpFile.deleteOnExit();
+        
+        InputStream mappingFileIn = null;
+        OutputStream mappingTmpOut = null;
+        
+        try {
+          mappingFileIn = LuceneSearchService.class.getResourceAsStream(
+              "/org/apache/opennlp/corpus_server/search/" + corpusId + ".xml");
+          mappingTmpOut = new FileOutputStream(mappingTmpFile);
+          
+          byte buffer[] = new byte[1024];
+          int len = 0;
+          while ((len = mappingFileIn.read(buffer)) > 0) {
+            mappingTmpOut.write(buffer, 0, len);
+          }
+        }
+        catch (IOException e) {
+          // TODO: Or just ignore it ?! and do not create the indexer for this corpus?!
+          throw e;
+        }
+        finally {
+          if (mappingFileIn != null) {
+            try {
+              mappingFileIn.close();
+            }
+            catch (IOException e) {}
+          }
+          
+          if (mappingTmpOut != null) {
+            try {
+              mappingTmpOut.close();
+            }
+            catch (IOException e) {}
+          }
+        }
+        
+        
+        specifier.getAnalysisEngineMetaData().
+        getConfigurationParameterSettings().setParameterValue("mappingFile",
+            mappingTmpFile.getAbsolutePath());
+        
+        // Set the index writer properties file in the analysis engine
+        // and replace the index path with the index location for this corpus
+        
+        Properties indexWriterProperties = new Properties();
+        
+        InputStream indexWriterPropertiesIn = null;
+        try {
+          // TODO: Retrieve file form somewhere for this corpus
+          indexWriterPropertiesIn = LuceneSearchService.class.getResourceAsStream(
+              "/org/apache/opennlp/corpus_server/search/IndexWriter.properties");
+        
+          indexWriterProperties.load(indexWriterPropertiesIn);
+        }
+        finally {
+          if (indexWriterPropertiesIn != null) {
+            try {
+              indexWriterPropertiesIn.close();
+            }
+            catch (IOException e) {}
+          }
+        }
+        
+        indexWriterProperties.setProperty("indexPath", getIndexDirectory(corpusId).getAbsolutePath());
+        
+        File indexWriterTmpFile = File.createTempFile("index-writer", corpusId + ".properties");
+        indexWriterTmpFile.deleteOnExit();
+        
+        OutputStream indexPropertiesOut = null; 
+        try {
+          indexPropertiesOut = new FileOutputStream(indexWriterTmpFile);
+          // write properties into a tmp file
+          indexWriterProperties.store(indexPropertiesOut, null);
+        }
+        finally {
+          if (indexPropertiesOut != null) {
+            try {
+              indexPropertiesOut.close();
+            }
+            catch (IOException e) {}
+          }
+        }
+        
+        FileResourceSpecifier indexWriterFileSpecifier = new FileResourceSpecifier_impl();
+        indexWriterFileSpecifier.setFileUrl(indexWriterTmpFile.toURL().toString());
+        // TODO: This will fail ...
+        specifier.getResourceManagerConfiguration().getExternalResources()[0].setResourceSpecifier(indexWriterFileSpecifier);
+        
+        AnalysisEngine indexer = UIMAFramework.produceAnalysisEngine(specifier);
+        corpusIndexerMap.put(corpusId, indexer);
+      } catch (InvalidXMLException e) {
+        throw new IOException(e);
+      } catch (ResourceInitializationException e) {
+        throw new IOException(e);
+      }
     }
   }
 
   
   @Override
-  public void index(CorpusStore store, String casId) throws IOException {
-    
-    // Get a reference to the indexer for this corpus ...
+  public  synchronized void index(CorpusStore store, String casId) throws IOException {
     
     // TODO: Need to take care for thread safety ..
     
-    // TODO: AE type system must be merged with corpus type system .. 
-    // CasCreationUtils.createCas(list);
-    // Does that also work with a type system description?!
-    List specs = new ArrayList();
-    specs.add(indexer); // Note: Is this allowed?
+    String corpusId = store.getCorpusId();
+    
+    AnalysisEngine indexer = corpusIndexerMap.get(corpusId);
+    
+    
+    InputStream indexTsIn = LuceneSearchService.class.getResourceAsStream(
+        "/org/apache/opennlp/corpus_server/search/TypeSystem.xml");
+    
+    TypeSystemDescription indexTypeDesc;
+    try {
+      indexTypeDesc = UimaUtil.createTypeSystemDescription(indexTsIn);
+    }
+    finally {
+      indexTsIn.close();
+    }
+    
+    List<MetaDataObject> specs = new ArrayList<MetaDataObject>();
+    specs.add(indexTypeDesc);
     specs.add(store.getTypeSystem());
     
+    // Note: This might be a performance problem
     CAS cas;
     try {
       cas = CasCreationUtils.createCas(specs);
@@ -102,6 +233,14 @@ public class LuceneSearchService impleme
     
     UimaUtil.deserializeXmiCAS(cas, new ByteArrayInputStream(casBytes));
     
+    // Inject id feature structure into the CAS
+    Type casIdType = cas.getTypeSystem().getType(LuceneIndexer.CAS_ID_TYPE);
+    Feature casIdFeature =  casIdType.getFeatureByBaseName(LuceneIndexer.CAS_ID_FEEATURE);
+
+    FeatureStructure casIdFS = cas.createFS(casIdType);
+    casIdFS.setStringValue(casIdFeature, casId);
+    cas.addFsToIndexes(casIdFS);
+    
     try {
       indexer.process(cas);
     } catch (AnalysisEngineProcessException e) {
@@ -115,15 +254,64 @@ public class LuceneSearchService impleme
   public Collection<String> search(CorpusStore store, String q)
       throws IOException {
     
+    // TODO:
+    // Creating a reader per query is can only be done for testing this ... 
+    // Open/Reopen index once in a while to see updates, can this be done automatically?!
+    
+    File indexLocation = getIndexDirectory(store.getCorpusId());
+    
+    QueryParser parser = null;
+    
+    final IndexSearcher searcher = new IndexSearcher(indexLocation.getAbsolutePath());
+    parser = new QueryParser("content", new StandardAnalyzer());
+    
+    Query query;
+    try {
+      query = parser.parse(q);
+    } catch (ParseException e) {
+      throw new IOException(e);
+    }
+    
+    final Set<String> results = new HashSet<String>();
+    
     // query index ...
+    searcher.search(query, new Collector() {
+      
+      @Override
+      public void setScorer(Scorer scorer) throws IOException {
+      }
+      
+      @Override
+      public void setNextReader(IndexReader reader, int arg1) throws IOException {
+      }
+      
+      @Override
+      public void collect(int id) throws IOException {
+        Document doc = searcher.doc(id);
+        String idString = doc.get("id");
+        results.add(idString);
+      }
+      
+      @Override
+      public boolean acceptsDocsOutOfOrder() {
+        return false;
+      }
+    });
+      
+    searcher.close();
     
-    return Collections.emptySet();
+    return results;
   }
 
   @Override
   public void shutdown() throws IOException {
-    if (indexer != null) {
-      indexer.destroy();
+    
+    for (String corpusId : corpusIndexerMap.keySet()) {
+      AnalysisEngine indexer = corpusIndexerMap.get(corpusId);
+      
+      if (indexer != null) {
+        indexer.destroy();
+      }
     }
   }
 }

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/SearchService.java Sun Aug 14 16:35:27 2011
@@ -20,11 +20,12 @@ package org.apache.opennlp.corpus_server
 import java.io.IOException;
 import java.util.Collection;
 
+import org.apache.opennlp.corpus_server.store.CorporaStore;
 import org.apache.opennlp.corpus_server.store.CorpusStore;
 
 public interface SearchService {
 
-  void initialize() throws IOException;
+  void initialize(CorporaStore corporaStore) throws IOException;
   
   // index
   void index(CorpusStore store, String casId) throws IOException;

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java Sun Aug 14 16:35:27 2011
@@ -18,6 +18,7 @@
 package org.apache.opennlp.corpus_server.store;
 
 import java.io.IOException;
+import java.util.Set;
 
 /**
  * The Corpora Store is the central storage which manages the corpora.
@@ -45,6 +46,8 @@ public interface CorporaStore {
    */
   void createCorpus(String corpusName, byte typeSystemBytes[]) throws IOException;
   
+  Set<String> getCorpusIds() throws IOException;
+  
   /**
    * Retrieves a corpus of the given name from the store.
    * 

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java Sun Aug 14 16:35:27 2011
@@ -27,6 +27,8 @@ import org.apache.uima.resource.metadata
  */
 public interface CorpusStore {
   
+  String getCorpusId();
+  
   byte[] getCAS(String casId) throws IOException;
   
   void addCAS(String casID, byte[] content) throws IOException;

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java Sun Aug 14 16:35:27 2011
@@ -20,10 +20,15 @@ package org.apache.opennlp.corpus_server
 import java.io.IOException;
 import java.sql.Blob;
 import java.sql.Connection;
+import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -86,6 +91,37 @@ public class DerbyCorporaStore extends A
     }
   }
 
+  public Set<String> getCorpusIds() throws IOException {
+    
+    Set<String> corpusIds = new HashSet<String>();
+    
+    try {
+      Connection conn = dataSource.getConnection();
+      DatabaseMetaData dbmd = conn.getMetaData();
+
+      String[] types = { "TABLE" };
+      ResultSet resultSet = dbmd.getTables(null, null, "%", types);
+
+      while (resultSet.next()) {
+        String tableName = resultSet.getString(3);
+        corpusIds.add(tableName.toLowerCase());
+      }
+
+      conn.close();
+
+    } catch (SQLException e) {
+
+      if (LOGGER.isLoggable(Level.SEVERE)) {
+        LOGGER.log(Level.SEVERE, "Failed to retrieve corpus ids!", e);
+      }
+
+      throw new IOException(e);
+    }
+    
+    
+    return Collections.unmodifiableSet(corpusIds); 
+  }
+  
   @Override
   public CorpusStore getCorpus(String corpusId) {
     return new DerbyCorpusStore(dataSource, this, corpusId);

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java Sun Aug 14 16:35:27 2011
@@ -40,7 +40,7 @@ public class DerbyCorpusStore implements
 
   private DataSource dataSource;
   private DerbyCorporaStore store;
-  private String corpusName;
+  private final String corpusName;
   
   DerbyCorpusStore(DataSource dataSource, DerbyCorporaStore store, String corpusName) {
     this.dataSource = dataSource;
@@ -49,6 +49,11 @@ public class DerbyCorpusStore implements
   }
   
   @Override
+  public String getCorpusId() {
+    return corpusName;
+  }
+  
+  @Override
   public byte[] getCAS(String casId) throws IOException {
     
     byte casBytes[]  = null;

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java Sun Aug 14 16:35:27 2011
@@ -18,8 +18,11 @@
 package org.apache.opennlp.corpus_server.store;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.opennlp.corpus_server.UimaUtil;
 
@@ -44,6 +47,11 @@ public class MemoryCorporaStore extends 
 		return instance;
 	}
 	
+	@Override
+	public Set<String> getCorpusIds() throws IOException {
+	  return Collections.unmodifiableSet(corpora.keySet());
+	}
+	
 	// Note: Add one twice, overwrites an existing one!
 	public void createCorpus(String corpusName,
 			byte typeSystemBytes[]) {

Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java Sun Aug 14 16:35:27 2011
@@ -41,6 +41,11 @@ public class MemoryCorpusStore implement
 		this.typeSystem = typeSystem;
 	}
 	
+	@Override
+	public String getCorpusId() {
+	  return corpusName;
+	}
+	
 	public byte[] getCAS(String casId) {
 		return casStore.get(casId);
 	}

Added: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties?rev=1157558&view=auto
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties (added)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties Sun Aug 14 16:35:27 2011
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The indexPath replaced by the analysis engine integration
+# indexPath = 
+#RAMBufferSize = 513
+#useCompoundFileFormat = false
+#maxFieldLength = 9999
+#uniqueIndex = true
\ No newline at end of file

Propchange: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/IndexWriter.properties
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml?rev=1157558&r1=1157557&r2=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml Sun Aug 14 16:35:27 2011
@@ -41,7 +41,7 @@
       <nameValuePair>
         <name>mappingFile</name>
         <value>
-          <string>/Users/joern/dev/opennlp-apache/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml</string>
+          <string>file-will-be-set-by-ae-integration</string>
         </value>
       </nameValuePair>
     </configurationParameterSettings>
@@ -61,4 +61,24 @@
       <outputsNewCASes>false</outputsNewCASes>
     </operationalProperties>
   </processingResourceMetaData>
+  
+  <resourceManagerConfiguration>
+    <externalResources>
+      <externalResource>
+        <name>LucasIndexWriterProvider</name>
+        <description>Provides a index writer instance.</description>
+        <fileResourceSpecifier>
+          <fileUrl>file:src/test/resources/IndexWriter.properties</fileUrl>
+        </fileResourceSpecifier>
+        <implementationName>org.apache.uima.lucas.indexer.IndexWriterProviderImpl</implementationName>
+      </externalResource>
+    </externalResources>
+    <externalResourceBindings>
+      <externalResourceBinding>
+        <key>indexWriterProvider</key>
+        <resourceName>LucasIndexWriterProvider</resourceName>
+      </externalResourceBinding>
+    </externalResourceBindings>
+  </resourceManagerConfiguration>
+  
 </analysisEngineDescription>

Added: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml?rev=1157558&view=auto
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml (added)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml Sun Aug 14 16:35:27 2011
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+	<!--
+	 ***************************************************************
+	 * Licensed to the Apache Software Foundation (ASF) under one
+	 * or more contributor license agreements.  See the NOTICE file
+	 * distributed with this work for additional information
+	 * regarding copyright ownership.  The ASF licenses this file
+	 * to you under the Apache License, Version 2.0 (the
+	 * "License"); you may not use this file except in compliance
+	 * with the License.  You may obtain a copy of the License at
+         *
+	 *   http://www.apache.org/licenses/LICENSE-2.0
+	 * 
+	 * Unless required by applicable law or agreed to in writing,
+	 * software distributed under the License is distributed on an
+	 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+	 * KIND, either express or implied.  See the License for the
+	 * specific language governing permissions and limitations
+	 * under the License.
+	 ***************************************************************
+   -->
+   
+<typeSystemDescription  xmlns="http://uima.apache.org/resourceSpecifier">
+	<name>IndexerTypeSystem</name>
+	<types>
+		<typeDescription>
+			<name>org.apache.opennlp.corpus_server.CasId</name>
+			<supertypeName>uima.cas.TOP</supertypeName>
+			<features>
+				<featureDescription>
+					<name>id</name>
+					<description></description>
+					<rangeTypeName>uima.cas.String</rangeTypeName>
+				</featureDescription>
+			</features>
+		</typeDescription>
+	</types>
+</typeSystemDescription>
\ No newline at end of file

Propchange: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/TypeSystem.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Copied: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml (from r1155976, incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml)
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml?p2=incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml&p1=incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml&r1=1155976&r2=1157558&rev=1157558&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/lucas.xml (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml Sun Aug 14 16:35:27 2011
@@ -22,7 +22,7 @@
 <fields xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:noNamespaceSchemaLocation="../../main/resources/lucas.xsd">
 
-	<field name="headline" index="yes" stored="yes">
+	<field name="headline" index="yes" stored="no">
 		<annotations>
 			<annotation type="org.apache.opennlp.annotations.Headline" tokenizer="standard"/>
 		</annotations>

Propchange: incubator/opennlp/sandbox/corpus-server/src/main/resources/org/apache/opennlp/corpus_server/search/wikinews.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain