You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by jo...@apache.org on 2009/07/24 10:58:55 UTC
svn commit: r797373 [2/8] - in /incubator/uima/sandbox/trunk/Lucas: ./
docbook/ docbook/LuceneCASConsumerUserGuide/
docbook/LuceneCASConsumerUserGuide/images/
src/main/java/org/apache/uima/lucas/consumer/
src/main/java/org/apache/uima/lucas/indexer/ sr...
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java Fri Jul 24 08:58:52 2009
@@ -19,281 +19,259 @@
package org.apache.uima.lucas.consumer;
-import java.io.BufferedReader;
-import java.io.FileReader;
+import java.io.File;
import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
+import org.apache.uima.UimaContext;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASException;
import org.apache.uima.collection.CasConsumer_ImplBase;
import org.apache.uima.jcas.JCas;
-import org.apache.uima.lucas.indexer.AnnotationDescription;
import org.apache.uima.lucas.indexer.AnnotationTokenStreamBuilder;
+import org.apache.uima.lucas.indexer.AnnotationTokenStreamBuildingException;
import org.apache.uima.lucas.indexer.DocumentBuilder;
import org.apache.uima.lucas.indexer.FieldBuilder;
-import org.apache.uima.lucas.indexer.FieldDescription;
+import org.apache.uima.lucas.indexer.FieldBuildingException;
import org.apache.uima.lucas.indexer.FilterBuilder;
-import org.apache.uima.lucas.indexer.IndexMappingFileReader;
+import org.apache.uima.lucas.indexer.FilterBuildingException;
+import org.apache.uima.lucas.indexer.IndexWriterProvider;
import org.apache.uima.lucas.indexer.Tokenizer;
-import org.apache.uima.lucas.indexer.util.MapFileReader;
-import org.apache.uima.lucas.indexer.util.MultimapFileReader;
-import org.apache.uima.lucas.indexer.util.PlainFileReader;
+import org.apache.uima.lucas.indexer.analysis.DefaultFilterFactoryRegistry;
+import org.apache.uima.lucas.indexer.analysis.TokenFilterFactory;
+import org.apache.uima.lucas.indexer.mapping.AnnotationDescription;
+import org.apache.uima.lucas.indexer.mapping.AnnotationMapper;
+import org.apache.uima.lucas.indexer.mapping.ElementMapper;
+import org.apache.uima.lucas.indexer.mapping.FeatureMapper;
+import org.apache.uima.lucas.indexer.mapping.FieldDescription;
+import org.apache.uima.lucas.indexer.mapping.FieldMapper;
+import org.apache.uima.lucas.indexer.mapping.FilterDescription;
+import org.apache.uima.lucas.indexer.mapping.FilterMapper;
+import org.apache.uima.lucas.indexer.mapping.MappingFileReader;
+import org.apache.uima.resource.ResourceAccessException;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.ResourceProcessException;
+import org.xml.sax.SAXException;
/**
- * Reads CAS object and writes the particular information in fields of a Lucene index. requires a
- * mapping file
+ * Reads CAS object and writes the particular information in fields of a Lucene index.
+ * requires a mapping file
*/
public class LuceneCASIndexer extends CasConsumer_ImplBase {
- private static final Logger logger = Logger.getLogger(LuceneCASIndexer.class);
+ private static final String RESOURCE_INDEX_WRITER_PROVIDER = "indexWriterProvider";
+
+ private static final Logger logger = Logger
+ .getLogger(LuceneCASIndexer.class);
- public final static String PARAM_OUTDIR = "indexOutDir";
+ public final static String PARAM_MAPPINGFILE = "mappingFile";
- public final static String PARAM_MAPPINGFILE = "mappingFile";
+ private IndexWriter indexWriter;
- public final static String PARAM_STOPWORDFILE = "stopwordFile";
+ private Collection<FieldDescription> fieldDescriptions;
+
+ private DocumentBuilder documentBuilder;
+
+ private FieldBuilder fieldBuilder;
+
+ private FilterBuilder filterBuilder;
+
+ private AnnotationTokenStreamBuilder annotationTokenStreamBuilder;
+
+ private Tokenizer tokenizer;
+
+ private Map<String, TokenFilterFactory> defaultFilterFactoryRegistry;
+
+ /**
+ * initializes the analyzer
+ */
+ public void initialize() throws ResourceInitializationException {
+ createFieldDescriptions();
+ getIndexWriterInstance();
+ createFilterBuilderWithPreloadedResources();
+
+ annotationTokenStreamBuilder = new AnnotationTokenStreamBuilder();
+ tokenizer = new Tokenizer();
+ fieldBuilder = new FieldBuilder(filterBuilder);
+ documentBuilder = new DocumentBuilder();
+ }
+
+ private void createFieldDescriptions()
+ throws ResourceInitializationException {
+ String mappingFilePath = (String) getConfigParameterValue(PARAM_MAPPINGFILE);
+
+ try {
+ MappingFileReader indexMappingFileReader = createMappingFileReader();
+ File mappingFile = new File(mappingFilePath);
+ fieldDescriptions = indexMappingFileReader
+ .readFieldDescriptionsFromFile(mappingFile);
+ } catch (IOException e) {
+ throw new ResourceInitializationException(e);
+ }
+ }
+
+ private MappingFileReader createMappingFileReader() throws IOException{
+ try {
+ SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+ Map<String, ElementMapper<?>> elementMappers = new HashMap<String, ElementMapper<?>>();
+ elementMappers.put(MappingFileReader.ANNOTATION, new AnnotationMapper());
+ elementMappers.put(MappingFileReader.FILTER, new FilterMapper());
+ elementMappers.put(MappingFileReader.FIELD, new FieldMapper());
+ elementMappers.put(MappingFileReader.FEATURE, new FeatureMapper());
+ return new MappingFileReader(parser, elementMappers);
+
+ } catch (ParserConfigurationException e) {
+ throw new IOException("Can't build SAXParser", e);
+ } catch (SAXException e) {
+ throw new IOException("Can't build SAXParser", e);
+ }
+ }
+
+ private void getIndexWriterInstance()
+ throws ResourceInitializationException {
+ UimaContext uimaContext = getUimaContext();
+ IndexWriterProvider indexWriterProvider;
+ try {
+ indexWriterProvider = (IndexWriterProvider) uimaContext
+ .getResourceObject(RESOURCE_INDEX_WRITER_PROVIDER);
+ } catch (ResourceAccessException e) {
+ throw new ResourceInitializationException(e);
+ }
+ indexWriter = indexWriterProvider.getIndexWriter();
+ }
+
+ private void createFilterBuilderWithPreloadedResources()
+ throws ResourceInitializationException {
+ defaultFilterFactoryRegistry = new DefaultFilterFactoryRegistry()
+ .getDefaultRegistry();
+ try {
+ preloadResources(fieldDescriptions, defaultFilterFactoryRegistry);
+ } catch (IOException e) {
+ throw new ResourceInitializationException(e);
+ }
+ filterBuilder = new FilterBuilder(defaultFilterFactoryRegistry);
+ }
+
+ void preloadResources(Collection<FieldDescription> fieldDescriptions,
+ Map<String, TokenFilterFactory> defaultFilterFactoryRegistry)
+ throws IOException {
+
+ for (FieldDescription fieldDescription : fieldDescriptions) {
+ for (FilterDescription filterDescription : fieldDescription
+ .getFilterDescriptions()) {
+ TokenFilterFactory tokenFilterFactory = defaultFilterFactoryRegistry
+ .get(filterDescription.getName());
+ if (tokenFilterFactory != null)
+ tokenFilterFactory.preloadResources(filterDescription
+ .getProperties());
+ }
+
+ for (AnnotationDescription annotationDescription : fieldDescription
+ .getAnnotationDescriptions()) {
+ for (FilterDescription filterDescription : annotationDescription
+ .getFilterDescriptions()) {
+ TokenFilterFactory tokenFilterFactory = defaultFilterFactoryRegistry
+ .get(filterDescription.getName());
+ if (tokenFilterFactory != null)
+ tokenFilterFactory.preloadResources(filterDescription
+ .getProperties());
+ }
+ }
+ }
+ }
+
+ public void processCas(CAS cas) throws ResourceProcessException {
+
+ try {
+ JCas jCas = cas.getJCas();
+ Collection<Field> fields = new ArrayList<Field>();
+ // iterate over field descriptions from mapping file
+ for (FieldDescription fieldDescription : fieldDescriptions) {
+ Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
+ // iterate over annotation descriptions
+ for (AnnotationDescription annotationDescription : fieldDescription
+ .getAnnotationDescriptions()) {
+ // create annotation token stream
+ TokenStream tokenStream = annotationTokenStreamBuilder
+ .createAnnotationTokenStream(jCas,
+ annotationDescription);
+
+ // needs (re)tokenization ?
+ if (tokenizer.needsTokenization(annotationDescription))
+ tokenStream = tokenizer.tokenize(tokenStream,
+ annotationDescription);
+
+ // wrap with filters
+ tokenStream = filterBuilder.filter(tokenStream,
+ annotationDescription.getFilterDescriptions());
+ tokenStreams.add(tokenStream);
+ }
+
+ // create fields
+ fields.addAll(fieldBuilder.createFields(tokenStreams,
+ fieldDescription));
+ }
+ // create document and add to index
+ Document document = documentBuilder.createDocument(fields);
+ indexWriter.addDocument(document);
+
+ } catch (AnnotationTokenStreamBuildingException e) {
+ logger.error("processCas(CAS)", e);
+ throw new ResourceProcessException(e);
+ } catch (IOException e) {
+ logger.error("processCas(CAS)", e);
+ throw new ResourceProcessException(e);
+ } catch (FieldBuildingException e) {
+ logger.error("processCas(CAS)", e);
+ throw new ResourceProcessException(e);
+ } catch (FilterBuildingException e) {
+ logger.error("processCas(CAS)", e);
+ throw new ResourceProcessException(e);
+ } catch (CASException e) {
+ logger.error("processCas(CAS)", e);
+ throw new ResourceProcessException(e);
+ }
+ }
+
+ @Override
+ public void destroy() {
+ logger.info("destroy " + LuceneCASIndexer.class);
+ optimizeIndex();
+ super.destroy();
+ }
+
+ public void optimizeIndex() {
+ try {
+ logger.info("optimizing the index now!");
+ indexWriter.optimize();
+ indexWriter.close();
+ } catch (IOException e) {
+ logger.error("exception while closing index", e);
+ }
+ }
+
+ FilterBuilder getFilterBuilder() {
+ return filterBuilder;
+ }
+
+ Collection<FieldDescription> getFieldDescriptions() {
+ return fieldDescriptions;
+ }
+
+ IndexWriter getIndexWriter() {
+ return indexWriter;
+ }
- public final static String PARAM_HYPERNYMFILE = "hypernymFile";
-
- public final static String PARAM_TOKEN_MAPPINGFILE = "tokenMappingFile";
-
- public final static String PARAM_UNIQUE_INDEX = "uniqueIndex";
-
- public final static String PARAM_RAM_BUFFER_SIZE = "ramBufferSize";
-
- public final static String PARAM_COMPOUND_FILE_FORMAT = "compoundFileFormat";
-
- public final static String PARAM_HEADER_TYPE_NAME = "headerTypeName";
-
- public final static String PARAM_DOC_ID_FEATURE_NAME = "docIdFeatureName";
-
- private IndexWriter indexWriter;
-
- private Collection<FieldDescription> fieldDescriptions;
-
- private DocumentBuilder documentBuilder;
-
- private FieldBuilder fieldBuilder;
-
- private FilterBuilder filterBuilder;
-
- private AnnotationTokenStreamBuilder annotationTokenStreamBuilder;
-
- private Tokenizer tokenizer;
-
- // private String headerTypeName;
- // private String docIdFeatureName;
-
- /**
- * initializes the analyzer
- */
- public void initialize() throws ResourceInitializationException {
-
- String mappingFile = (String) getConfigParameterValue(PARAM_MAPPINGFILE);
- String stopwordFile = (String) getConfigParameterValue(PARAM_STOPWORDFILE);
- String hypernymFile = (String) getConfigParameterValue(PARAM_HYPERNYMFILE);
- String[] mappingFiles = (String[]) getConfigParameterValue(PARAM_TOKEN_MAPPINGFILE);
- Boolean uniqueIndex = (Boolean) getConfigParameterValue(PARAM_UNIQUE_INDEX);
- String outDir = (String) getConfigParameterValue(PARAM_OUTDIR);
-
- // headerTypeName = (String) getConfigParameterValue(PARAM_HEADER_TYPE_NAME);
- // docIdFeatureName = (String) getConfigParameterValue(PARAM_DOC_ID_FEATURE_NAME);
-
- Integer ramBufferSize = (Integer) getConfigParameterValue(PARAM_RAM_BUFFER_SIZE);
- Boolean compoundFileFormat = (Boolean) getConfigParameterValue(PARAM_COMPOUND_FILE_FORMAT);
-
- try {
- if (uniqueIndex)
- outDir += "-" + getHostName() + "-" + getPID();
-
- IndexMappingFileReader indexMappingFileReader = new IndexMappingFileReader();
- fieldDescriptions = indexMappingFileReader.readFieldDescriptionsFromFile(mappingFile);
-
- indexWriter = new IndexWriter(outDir, new StandardAnalyzer());
- if (ramBufferSize != null)
- indexWriter.setRAMBufferSizeMB(ramBufferSize);
- if (compoundFileFormat != null)
- indexWriter.setUseCompoundFile(compoundFileFormat);
-
- String[] stopwords = null;
- Map<String, Map<String, String>> mappings = null;
- Map<String, List<String>> hypernyms = null;
-
- if (stopwordFile != null) {
- PlainFileReader stopwordReader =
- new PlainFileReader(new BufferedReader(new FileReader(stopwordFile)));
- stopwords = stopwordReader.readLines();
- } else
- stopwords = new String[0];
-
- if (hypernymFile != null) {
- MultimapFileReader hypernymReader =
- new MultimapFileReader(new BufferedReader(new FileReader(hypernymFile)));
- hypernyms = hypernymReader.readMultimap();
- } else
- hypernyms = Collections.EMPTY_MAP;
-
- if (mappingFiles != null && mappingFiles.length != 0)
- mappings = readMappingFiles(mappingFiles);
- else
- mappings = Collections.EMPTY_MAP;
-
- annotationTokenStreamBuilder = new AnnotationTokenStreamBuilder();
- tokenizer = new Tokenizer();
- filterBuilder = new FilterBuilder(stopwords, hypernyms, mappings);
- fieldBuilder = new FieldBuilder();
- documentBuilder = new DocumentBuilder();
-
- } catch (Throwable e) {
- logger.error("initialize()", e);
- throw new ResourceInitializationException(e);
- }
- }
-
- private Map<String, Map<String, String>> readMappingFiles(String[] mappingFiles)
- throws IOException {
- Map<String, Map<String, String>> mappings = new HashMap<String, Map<String, String>>();
- for (String mappingFile : mappingFiles) {
- MapFileReader reader = new MapFileReader(new BufferedReader(new FileReader(mappingFile)));
- Map<String, String> mapping = reader.readMap();
-
- if (mappingFile.lastIndexOf("/") != -1)
- mappingFile = mappingFile.substring(mappingFile.lastIndexOf("/") + 1);
-
- mappings.put(mappingFile, mapping);
- reader.close();
- }
-
- return mappings;
- }
-
- /**
- * optimizes the index
- *
- */
- public void optimizeIndex() {
- try {
- logger.info("optimizing the index now!");
- indexWriter.optimize();
- indexWriter.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void destroy() {
- logger.info("destroy " + LuceneCASIndexer.class);
- optimizeIndex();
- super.destroy();
- }
-
- /**
- * processes the CAS
- */
- public void processCas(CAS cas) throws ResourceProcessException {
- JCas jCas = null;
- try {
- jCas = cas.getJCas();
- } catch (CASException e) {
- logger.error("processCas(CAS)", e);
- }
-
- /*
- * if( headerTypeName != null && docIdFeatureName != null ){ String docId = getDocID(jCas,
- * headerTypeName, docIdFeatureName); logger.info("Processing document "+ docId ); }
- */
-
- try {
- Collection<Field> fields = new ArrayList<Field>();
- // iterate over field descriptions from mapping file
- for (FieldDescription fieldDescription : fieldDescriptions) {
- Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
- // iterate over annotation descriptions
- for (AnnotationDescription annotationDescription : fieldDescription
- .getAnnotationDescriptions()) {
- // create annotation token stream
- TokenStream tokenStream =
- annotationTokenStreamBuilder.createAnnotationTokenStream(jCas,
- annotationDescription);
-
- // needs (re)tokenization ?
- if (tokenizer.needsTokenization(annotationDescription))
- tokenStream = tokenizer.tokenize(tokenStream, annotationDescription);
-
- // wrap with filters
- tokenStream = filterBuilder.filter(tokenStream, annotationDescription);
- tokenStreams.add(tokenStream);
- }
-
- // create fields
- fields.addAll(fieldBuilder.createFields(tokenStreams, fieldDescription));
- }
- // create document and add to index
- Document document = documentBuilder.createDocument(fields);
- indexWriter.addDocument(document);
-
- } catch (Throwable e) {
- logger.error("processCas(CAS)", e); //$NON-NLS-1$
- throw new ResourceProcessException(e);
- }
- }
-
- protected FilterBuilder getFilterBuilder() {
- return filterBuilder;
- }
-
- protected Collection<FieldDescription> getFieldDescriptions() {
- return fieldDescriptions;
- }
-
- protected IndexWriter getIndexWriter() {
- return indexWriter;
- }
-
- protected String getPID() {
- String id = ManagementFactory.getRuntimeMXBean().getName();
- return id.substring(0, id.indexOf("@"));
- }
-
- public String getHostName() {
- InetAddress address;
- String hostName;
- try {
- address = InetAddress.getLocalHost();
- hostName = address.getHostName();
- } catch (UnknownHostException e) {
- throw new IllegalStateException(e);
- }
- return hostName;
- }
-
- /*
- * public String getDocID(JCas cas, String headerTypeName, String docIdFeatureName){
- *
- * Type headerType = cas.getTypeSystem().getType(headerTypeName); if( headerType != null ){
- * FSIterator headerIterator = cas.getAnnotationIndex(headerType).iterator(); if(
- * headerIterator.hasNext() ){ FeatureStructure headerFeatureStructure = headerIterator.get();
- * Feature docIdFeature = headerType.getFeatureByBaseName(docIdFeatureName); return
- * headerFeatureStructure.getFeatureValueAsString(docIdFeature); } }
- *
- * return null; }
- */
-}
+}
\ No newline at end of file
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java Fri Jul 24 08:58:52 2009
@@ -22,22 +22,29 @@
import java.text.DecimalFormat;
import java.text.Format;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.log4j.Logger;
import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.CASException;
import org.apache.uima.jcas.JCas;
import org.apache.uima.lucas.indexer.analysis.AnnotationTokenStream;
+import org.apache.uima.lucas.indexer.analysis.InvalidTokenSourceException;
+import org.apache.uima.lucas.indexer.mapping.AnnotationDescription;
+import org.apache.uima.lucas.indexer.mapping.FeatureDescription;
/**
* Builds annotation token streams wrapped with filters according to annotation descriptions.
*/
public class AnnotationTokenStreamBuilder {
+ private static final Logger LOGGER = Logger.getLogger(AnnotationTokenStreamBuilder.class);
+
+
public AnnotationTokenStream createAnnotationTokenStream(JCas jCas,
- AnnotationDescription annotationDescription) throws CASException {
+ AnnotationDescription annotationDescription) throws AnnotationTokenStreamBuildingException {
String sofaName = annotationDescription.getSofa();
sofaName = sofaName == null ? CAS.NAME_DEFAULT_SOFA : sofaName;
@@ -47,33 +54,64 @@
AnnotationTokenStream annotationTokenStream = null;
String typeName = annotationDescription.getType();
- if (annotationDescription.getFeatureDescriptions().size() == 0) // use coveredText
- annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName);
- else {
+ if (annotationDescription.getFeatureDescriptions().size() == 0)
+ try {
+ annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName);
+ } catch (InvalidTokenSourceException e) {
+ throw createException(annotationDescription, e);
+ }
+ else {
String featurePath = annotationDescription.getFeaturePath();
- String delimiter = annotationDescription.getConcatString();
+ String delimiter = annotationDescription.getFeatureValueDelimiterString();
+
+ Collection<FeatureDescription> featureDescriptions = annotationDescription.getFeatureDescriptions();
+ List<String> featureNames = extractFeatureNames(featureDescriptions);
+ Map<String, Format> featureFormats = extractFeatureFormats(featureDescriptions);
+ if (featurePath != null)
+ try {
+ annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName, featurePath,
+ featureNames, delimiter, featureFormats);
+ } catch (InvalidTokenSourceException e) {
+ throw createException(annotationDescription, e);
+ }
+ else
+ try {
+ annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName, featureNames,
+ delimiter, featureFormats);
+ } catch (InvalidTokenSourceException e) {
+ throw createException(annotationDescription, e);
+ }
+
+ }
+
+ return annotationTokenStream;
+ }
+
+ private List<String> extractFeatureNames(Collection<FeatureDescription> featureDescriptions){
List<String> featureNames = new ArrayList<String>();
- Map<String, Format> featureFormats = new HashMap<String, Format>();
- for (FeatureDescription featureDescription : annotationDescription.getFeatureDescriptions()) {
+ for (FeatureDescription featureDescription : featureDescriptions) {
featureNames.add(featureDescription.getFeatureName());
+ }
+ return featureNames;
+ }
+
+ private Map<String, Format> extractFeatureFormats(Collection<FeatureDescription> featureDescriptions){
+ Map<String, Format> featureFormats = new HashMap<String, Format>();
+ for (FeatureDescription featureDescription : featureDescriptions) {
if (featureDescription.getNumberFormat() != null) {
Format format = new DecimalFormat(featureDescription.getNumberFormat());
featureFormats.put(featureDescription.getFeatureName(), format);
}
}
-
- if (featurePath != null)
- annotationTokenStream =
- new AnnotationTokenStream(jCas, sofaName, typeName, featurePath, featureNames,
- delimiter, featureFormats);
- else
- annotationTokenStream =
- new AnnotationTokenStream(jCas, sofaName, typeName, featureNames, delimiter,
- featureFormats);
-
- }
-
- return annotationTokenStream;
+ return featureFormats;
}
+
+ private AnnotationTokenStreamBuildingException createException(AnnotationDescription annotationDescription,
+ InvalidTokenSourceException e) {
+ String message = "Can't build TokenStream from annotation at line " +
+ annotationDescription.getLineNumber();
+ return new AnnotationTokenStreamBuildingException(message, e);
+ }
+
}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+public class AnnotationTokenStreamBuildingException extends Exception {
+
+ public AnnotationTokenStreamBuildingException(String message){
+ super(message);
+ }
+
+ public AnnotationTokenStreamBuildingException(String message, Throwable cause){
+ super(message, cause);
+ }
+
+ public AnnotationTokenStreamBuildingException(Throwable cause){
+ super(cause);
+ }
+
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java Fri Jul 24 08:58:52 2009
@@ -19,16 +19,18 @@
package org.apache.uima.lucas.indexer;
-import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.uima.cas.CASException;
public class DocumentBuilder {
- public Document createDocument(Collection<Field> fields) throws CASException, IOException {
+ public DocumentBuilder() {
+ super();
+ }
+
+ public Document createDocument(Collection<Field> fields) {
Document document = new Document();
for (Field field : fields)
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java Fri Jul 24 08:58:52 2009
@@ -29,147 +29,205 @@
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
-import org.apache.uima.cas.CASException;
import org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator;
import org.apache.uima.lucas.indexer.analysis.TokenStreamMerger;
+import org.apache.uima.lucas.indexer.mapping.FieldDescription;
+import org.apache.uima.lucas.indexer.mapping.FilterDescription;
import org.apache.uima.lucas.indexer.util.TokenStreamStringConcatenator;
public class FieldBuilder {
- public static final String FIELD_INDEX_NO = "no";
+ public static final String FIELD_INDEX_NO = "no";
- public static final String FIELD_INDEX_YES = "yes";
+ public static final String FIELD_INDEX_YES = "yes";
- public static final String FIELD_INDEX_NO_NORMS = "no_norms";
+ public static final String FIELD_INDEX_NO_NORMS = "no_norms";
- public static final String FIELD_INDEX_NO_TF = "no_tf";
+ public static final String FIELD_INDEX_NO_TF = "no_tf";
- public static final String FIELD_INDEX_NO_NORMS_TF = "no_norms_tf";
-
- public static final String FIELD_TERM_VECTOR_NO = "no";
-
- public static final String FIELD_TERM_VECTOR_YES = "yes";
-
- public static final String FIELD_TERM_VECTOR_WITH_OFFSETS = "offsets";
-
- public static final String FIELD_TERM_VECTOR_WITH_POSITIONS = "positions";
-
- public static final String FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS = "positions_offsets";
-
- public static final String FIELD_STORE_NO = "no";
-
- public static final String FIELD_STORE_YES = "yes";
-
- public static final String FIELD_STORE_COMPRESS = "compress";
-
- private TokenStreamStringConcatenator tokenStreamStringConcatenator;
-
- public FieldBuilder() {
- tokenStreamStringConcatenator = new TokenStreamStringConcatenator();
- }
-
- public Collection<Field> createFields(Collection<TokenStream> tokenStreams,
- FieldDescription fieldDescription) throws IOException, CASException {
-
- TokenStream tokenStream = null;
- if (fieldDescription.getMerge())
- tokenStream = new TokenStreamMerger(tokenStreams);
- else if (tokenStreams.size() > 1)
- tokenStream = new TokenStreamConcatenator(tokenStreams);
- else if (tokenStreams.size() == 1)
- tokenStream = tokenStreams.iterator().next();
-
- String fieldName = fieldDescription.getName();
- String delimiter = fieldDescription.getDelimiter();
- Collection<Field> fields = new ArrayList<Field>();
-
- Field.Store fieldStore = getFieldStore(fieldDescription.getStored());
- Field.Index fieldIndex = getFieldIndex(fieldDescription.getIndex());
- Field.TermVector fieldTermVector = getFieldTermVector(fieldDescription.getTermVector());
-
- if (fieldStore == Field.Store.YES || fieldStore == Field.Store.COMPRESS)
- fields.addAll(createStoredFields(fieldName, tokenStream, fieldStore, delimiter));
-
- boolean omitTF =
- fieldDescription.getIndex().equals(FIELD_INDEX_NO_TF)
- || fieldDescription.getIndex().equals(FIELD_INDEX_NO_NORMS_TF);
-
- if (fieldIndex != Field.Index.NO)
- fields.add(createIndexedField(fieldName, tokenStream, fieldIndex, fieldTermVector, omitTF));
-
- return fields;
- }
-
- private Field createIndexedField(String fieldName, TokenStream tokenStream, Index fieldIndex,
- TermVector fieldTermVector, boolean omitTF) {
-
- Field field = new Field(fieldName, tokenStream, fieldTermVector);
- if (fieldIndex == Field.Index.NOT_ANALYZED_NO_NORMS)
- field.setOmitNorms(true);
- field.setOmitTf(omitTF);
-
- return field;
- }
-
- protected Collection<Field> createStoredFields(String fieldName, TokenStream tokenStream,
- Store fieldStore, String delimiter) throws IOException {
- Collection<Field> fields = new ArrayList<Field>();
- if (delimiter != null) {
- String value =
- tokenStreamStringConcatenator
- .tokenStreamToStringWithDelimiter(tokenStream, delimiter);
- fields.add(new Field(fieldName, value, fieldStore, Field.Index.NO));
- } else {
- Token nextToken = tokenStream.next(new Token());
- while (nextToken != null) {
- String value = new String(nextToken.termBuffer(), 0, nextToken.termLength());
- fields.add(new Field(fieldName, value, fieldStore, Field.Index.NO));
- nextToken = tokenStream.next(nextToken);
- }
- }
- tokenStream.reset();
-
- return fields;
- }
-
- protected Field.Index getFieldIndex(String index) {
- if (index.equals(FIELD_INDEX_NO))
- return Field.Index.NO;
- else if (index.equals(FIELD_INDEX_YES))
- return Field.Index.NOT_ANALYZED;
- else if (index.equals(FIELD_INDEX_NO_NORMS))
- return Field.Index.NOT_ANALYZED_NO_NORMS;
- else if (index.equals(FIELD_INDEX_NO_NORMS_TF))
- return Field.Index.NOT_ANALYZED_NO_NORMS;
- else if (index.equals(FIELD_INDEX_NO_TF))
- return Field.Index.NOT_ANALYZED;
-
- throw new IllegalArgumentException("unknown index parameter: " + index);
- }
-
- protected Field.TermVector getFieldTermVector(String termVector) {
- if (termVector.equals(FIELD_TERM_VECTOR_NO))
- return Field.TermVector.NO;
- else if (termVector.equals(FIELD_TERM_VECTOR_YES))
- return Field.TermVector.YES;
- else if (termVector.equals(FIELD_TERM_VECTOR_WITH_OFFSETS))
- return Field.TermVector.WITH_OFFSETS;
- else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS))
- return Field.TermVector.WITH_POSITIONS;
- else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS))
- return Field.TermVector.WITH_POSITIONS_OFFSETS;
-
- throw new IllegalArgumentException("unknown termVector parameter: " + termVector);
- }
-
- protected Field.Store getFieldStore(String store) {
- if (store.equals(FIELD_STORE_NO))
- return Field.Store.NO;
- else if (store.equals(FIELD_STORE_YES))
- return Field.Store.YES;
- else if (store.equals(FIELD_STORE_COMPRESS))
- return Field.Store.COMPRESS;
+ public static final String FIELD_INDEX_NO_NORMS_TF = "no_norms_tf";
+
+ public static final String FIELD_TERM_VECTOR_NO = "no";
+
+ public static final String FIELD_TERM_VECTOR_YES = "yes";
+
+ public static final String FIELD_TERM_VECTOR_WITH_OFFSETS = "offsets";
+
+ public static final String FIELD_TERM_VECTOR_WITH_POSITIONS = "positions";
+
+ public static final String FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS = "positions_offsets";
+
+ public static final String FIELD_STORE_NO = "no";
+
+ public static final String FIELD_STORE_YES = "yes";
+
+ public static final String FIELD_STORE_COMPRESS = "compress";
+
+ private TokenStreamStringConcatenator tokenStreamStringConcatenator;
+ private FilterBuilder filterBuilder;
+ private FieldDescription fieldDescription;
+
+ public FieldBuilder(FilterBuilder filterBuilder) {
+ tokenStreamStringConcatenator = new TokenStreamStringConcatenator();
+ this.filterBuilder = filterBuilder;
+ }
+
+ public Collection<Field> createFields(Collection<TokenStream> tokenStreams,
+ FieldDescription fieldDescription) throws FieldBuildingException {
+
+ TokenStream tokenStream = createFieldTokenStream(tokenStreams,
+ fieldDescription);
+
+ this.fieldDescription = fieldDescription;
+ String fieldName = fieldDescription.getName();
+ String delimiter = fieldDescription.getDelimiter();
+ Collection<Field> fields = new ArrayList<Field>();
+
+ Collection<FilterDescription> filterDescriptions = fieldDescription
+ .getFilterDescriptions();
+ tokenStream = getFilteredTokenStream(fieldName, tokenStream,
+ filterDescriptions);
+
+ Field.Store fieldStore = getFieldStore(fieldDescription.getStored());
+ Field.Index fieldIndex = getFieldIndex(fieldDescription.getIndex());
+ Field.TermVector fieldTermVector = getFieldTermVector(fieldDescription
+ .getTermVector());
+
+ if (fieldStore == Field.Store.YES || fieldStore == Field.Store.COMPRESS)
+ fields.addAll(createStoredFields(fieldName, tokenStream,
+ fieldStore, delimiter));
+
+ boolean omitTF = fieldDescription.getIndex().equals(FIELD_INDEX_NO_TF)
+ || fieldDescription.getIndex().equals(FIELD_INDEX_NO_NORMS_TF);
+
+ if (fieldIndex != Field.Index.NO)
+ fields.add(createIndexedField(fieldName, tokenStream, fieldIndex,
+ fieldTermVector, omitTF));
+
+ return fields;
+ }
+
+ private TokenStream createFieldTokenStream(
+ Collection<TokenStream> tokenStreams,
+ FieldDescription fieldDescription) throws FieldBuildingException
+ {
+ TokenStream tokenStream = null;
+ if (fieldDescription.getMerge())
+ tokenStream = getTokenStreamMerger(tokenStreams);
+ else if (tokenStreams.size() > 1)
+ tokenStream = new TokenStreamConcatenator(tokenStreams);
+ else if (tokenStreams.size() == 1)
+ tokenStream = tokenStreams.iterator().next();
+ return tokenStream;
+ }
+
+ private TokenStream getFilteredTokenStream(String fieldName,
+ TokenStream tokenStream,
+ Collection<FilterDescription> filterDescriptions)
+ throws FieldBuildingException {
+ try {
+ return filterBuilder.filter(tokenStream, filterDescriptions);
+ } catch (FilterBuildingException e) {
+ throw createException(e);
+ }
+ }
+
+ private FieldBuildingException createException(Exception e) {
+ String message = "Can't build field " + fieldDescription.getName()+
+ " at line " + fieldDescription.getLineNumber();
+ return new FieldBuildingException(message, e);
+ }
+
+ private TokenStream getTokenStreamMerger(
+ Collection<TokenStream> tokenStreams) throws FieldBuildingException {
+ try {
+ return new TokenStreamMerger(tokenStreams);
+ } catch (IOException e) {
+ throw createException(e);
+ }
+
+ }
+
+ private Field createIndexedField(String fieldName, TokenStream tokenStream,
+ Index fieldIndex, TermVector fieldTermVector, boolean omitTF) {
+
+ Field field = new Field(fieldName, tokenStream, fieldTermVector);
+ if (fieldIndex == Field.Index.NOT_ANALYZED_NO_NORMS)
+ field.setOmitNorms(true);
+ field.setOmitTf(omitTF);
+
+ return field;
+ }
+
+ protected Collection<Field> createStoredFields(String fieldName,
+ TokenStream tokenStream, Store fieldStore, String delimiter)
+ throws FieldBuildingException {
+ Collection<Field> fields = new ArrayList<Field>();
+ try {
+ if (delimiter != null) {
+ String value = tokenStreamStringConcatenator
+ .tokenStreamToStringWithDelimiter(tokenStream,
+ delimiter);
+ fields.add(new Field(fieldName, value, fieldStore,
+ Field.Index.NO));
+ } else {
+ Token nextToken = tokenStream.next(new Token());
+ while (nextToken != null) {
+ String value = new String(nextToken.termBuffer(), 0,
+ nextToken.termLength());
+ fields.add(new Field(fieldName, value, fieldStore,
+ Field.Index.NO));
+ nextToken = tokenStream.next(nextToken);
+ }
+ }
+ tokenStream.reset();
+ } catch (IOException e) {
+ throw createException(e);
+ }
+ return fields;
+ }
+
+ protected Field.Index getFieldIndex(String index) throws FieldBuildingException {
+ if (index.equals(FIELD_INDEX_NO))
+ return Field.Index.NO;
+ else if (index.equals(FIELD_INDEX_YES))
+ return Field.Index.NOT_ANALYZED;
+ else if (index.equals(FIELD_INDEX_NO_NORMS))
+ return Field.Index.NOT_ANALYZED_NO_NORMS;
+ else if (index.equals(FIELD_INDEX_NO_NORMS_TF))
+ return Field.Index.NOT_ANALYZED_NO_NORMS;
+ else if (index.equals(FIELD_INDEX_NO_TF))
+ return Field.Index.NOT_ANALYZED;
+
+ throw createException(new IllegalArgumentException("unknown index parameter: " + index));
+
+ }
+
+ protected Field.TermVector getFieldTermVector(String termVector) throws FieldBuildingException {
+ if (termVector.equals(FIELD_TERM_VECTOR_NO))
+ return Field.TermVector.NO;
+ else if (termVector.equals(FIELD_TERM_VECTOR_YES))
+ return Field.TermVector.YES;
+ else if (termVector.equals(FIELD_TERM_VECTOR_WITH_OFFSETS))
+ return Field.TermVector.WITH_OFFSETS;
+ else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS))
+ return Field.TermVector.WITH_POSITIONS;
+ else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS))
+ return Field.TermVector.WITH_POSITIONS_OFFSETS;
+
+ throw createException(new IllegalArgumentException("unknown termVector parameter: "
+ + termVector));
+ }
+
+ protected Field.Store getFieldStore(String store) throws FieldBuildingException {
+ if (store.equals(FIELD_STORE_NO))
+ return Field.Store.NO;
+ else if (store.equals(FIELD_STORE_YES))
+ return Field.Store.YES;
+ else if (store.equals(FIELD_STORE_COMPRESS))
+ return Field.Store.COMPRESS;
+
+ throw createException(new IllegalArgumentException("unknown stored parameter: " + store));
+ }
- throw new IllegalArgumentException("unknown stored parameter: " + store);
- }
}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+public class FieldBuildingException extends Exception {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public FieldBuildingException(String message) {
+ super(message);
+ }
+
+ public FieldBuildingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public FieldBuildingException(Throwable cause) {
+ super(cause);
+ }
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java Fri Jul 24 08:58:52 2009
@@ -20,115 +20,141 @@
package org.apache.uima.lucas.indexer;
import java.io.IOException;
-import java.util.List;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
import java.util.Map;
import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.snowball.SnowballFilter;
-import org.apache.uima.lucas.indexer.analysis.AdditionTokenFilter;
-import org.apache.uima.lucas.indexer.analysis.HypernymTokenFilter;
-import org.apache.uima.lucas.indexer.analysis.PositionFilter;
-import org.apache.uima.lucas.indexer.analysis.ReplaceFilter;
-import org.apache.uima.lucas.indexer.analysis.SplitterFilter;
-import org.apache.uima.lucas.indexer.analysis.UniqueFilter;
-import org.apache.uima.lucas.indexer.analysis.UpperCaseTokenFilter;
+import org.apache.uima.lucas.indexer.analysis.TokenFilterFactory;
+import org.apache.uima.lucas.indexer.mapping.FilterDescription;
+import org.apache.uima.lucas.indexer.mapping.Locateable;
public class FilterBuilder {
- private String[] stopwords;
-
- private Map<String, List<String>> hypernyms;
-
- private Map<String, Map<String, String>> mappings;
-
public static final Logger LOGGER = Logger.getLogger(FilterBuilder.class);
- public static final String POSITION_FIRST = "first";
-
- public static final String POSITION_LAST = "last";
-
- public FilterBuilder(String[] stopwords, Map<String, List<String>> hypernyms,
- Map<String, Map<String, String>> mappings) {
- super();
- this.stopwords = stopwords;
- this.hypernyms = hypernyms;
- this.mappings = mappings;
- }
-
- public TokenStream filter(TokenStream tokenStream, AnnotationDescription description)
- throws IOException {
-
- String mappingFile = description.getMappingFile();
- if (mappingFile != null) {
- Map<String, String> mapping = mappings.get(mappingFile);
- if (mapping == null)
- throw new IllegalStateException(mappingFile + " not found!");
- tokenStream = new ReplaceFilter(tokenStream, mapping);
- }
-
- // add filters
- if (description.getPosition() != null) {
- String position = description.getPosition();
- if (position.equals(AnnotationDescription.FIRST_POSITION))
- tokenStream = new PositionFilter(tokenStream, PositionFilter.FIRST_POSITION);
- else if (position.equals(AnnotationDescription.LAST_POSITION))
- tokenStream = new PositionFilter(tokenStream, PositionFilter.LAST_POSITION);
- }
-
- String prefix = description.getPrefix();
- if (prefix != null)
- tokenStream = new AdditionTokenFilter(tokenStream, prefix, AdditionTokenFilter.PREFIX);
-
- String postfix = description.getPostfix();
- if (postfix != null)
- tokenStream = new AdditionTokenFilter(tokenStream, postfix, AdditionTokenFilter.POSTFIX);
-
- String splitString = description.getSplitString();
- if (splitString != null) {
- tokenStream = new SplitterFilter(tokenStream, splitString);
- }
+ private Map<String, TokenFilterFactory> cachedFactories;
+ private Locateable currentLocateable;
+
+ public FilterBuilder(Map<String, TokenFilterFactory> predefinedFactories) {
+ cachedFactories = predefinedFactories;
+ }
- if (description.getLowercase()) {
- tokenStream = new LowerCaseFilter(tokenStream);
- }
+ public TokenStream filter(TokenStream tokenStream,
+ Collection<FilterDescription> filterDescriptions) throws FilterBuildingException {
- if (description.getUppercase()) {
- tokenStream = new UpperCaseTokenFilter(tokenStream);
+ TokenStream filteredTokenStream = tokenStream;
+ for (FilterDescription filterDescription : filterDescriptions) {
+ String filterName = filterDescription.getName();
+ String factoryClassName = filterDescription.getFactoryClassName();
+ String className = filterDescription.getClassName();
+ currentLocateable = filterDescription;
+
+ // use cached factory to create filter
+ if (filterName != null && factoryClassName == null) {
+ TokenFilterFactory tokenFilterFactory = cachedFactories.get(filterName);
+ if (tokenFilterFactory == null)
+ throw createException("No factory registered for " + filterName
+ + ". Please provide a factory");
+
+ filteredTokenStream = createTokenFilter(filteredTokenStream, filterDescription, tokenFilterFactory);
+ }
+ // use single argument constructor of a token filter
+ else if (className != null)
+ {
+ filteredTokenStream = createTokenFilterWithClassName(filteredTokenStream, className);
+ }
+ // use factory for creating the filter and cache it if needed
+ else if (factoryClassName != null)
+ {
+ TokenFilterFactory tokenFilterFactory = createTokenFilterFactory(factoryClassName);
+ filteredTokenStream = createTokenFilter(filteredTokenStream, filterDescription, tokenFilterFactory);
+ if( filterDescription.isReuseFactory() ){
+ if( filterName == null )
+ throw createException("Provide a name for factory reuse ");
+ cachedFactories.put(filterName, tokenFilterFactory);
+ }
+ }
}
- if (description.getStopwordRemove()) {
- tokenStream = new StopFilter(tokenStream, stopwords);
- }
- if (description.getAddHypernyms()) {
- if (hypernyms.size() == 0)
- LOGGER.warn("hypernym adding activatet, but no hypernyms found!");
+ return filteredTokenStream;
+ }
- tokenStream = new HypernymTokenFilter(tokenStream, hypernyms);
- }
- String snowballFilter = description.getSnowballFilter();
- if (snowballFilter != null) {
- tokenStream = new SnowballFilter(tokenStream, snowballFilter);
+ private TokenFilter createTokenFilter(TokenStream filteredTokenStream,
+ FilterDescription filterDescription, TokenFilterFactory tokenFilterFactory)
+ throws FilterBuildingException {
+ try {
+ return tokenFilterFactory.createTokenFilter(filteredTokenStream,
+ filterDescription.getProperties());
+ } catch (IOException e) {
+ throw createException("Can't build filter with description", e);
}
+ }
- if (description.getUnique()) {
- tokenStream = new UniqueFilter(tokenStream);
+ private TokenFilterFactory createTokenFilterFactory(String factoryClassName)
+ throws FilterBuildingException {
+ try {
+ Class<?> clazz = Class.forName(factoryClassName);
+ Constructor<?>[] constructors = clazz.getConstructors();
+ for (Constructor<?> constructor : constructors) {
+ Class<?>[] parameters = constructor.getParameterTypes();
+ if (parameters.length == 0)
+ return (TokenFilterFactory) constructor.newInstance();
+ }
+ } catch (ClassNotFoundException e) {
+ throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+ } catch (IllegalArgumentException e) {
+ throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+ } catch (InstantiationException e) {
+ throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+ } catch (IllegalAccessException e) {
+ throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+ } catch (InvocationTargetException e) {
+ throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
}
+ throw new FilterBuildingException("Class " + factoryClassName + " has no public no argument constructor!");
+ }
- return tokenStream;
+ private TokenFilter createTokenFilterWithClassName(TokenStream tokenStream, String className)
+ throws FilterBuildingException {
+ try {
+ Class<?> clazz = Class.forName(className);
+ Constructor<?>[] constructors = clazz.getConstructors();
+ for (Constructor<?> constructor : constructors) {
+ Class<?>[] parameters = constructor.getParameterTypes();
+ if (parameters.length == 1 && parameters[0].equals(TokenStream.class))
+ return (TokenFilter) constructor.newInstance(tokenStream);
+ }
+ } catch (ClassNotFoundException e) {
+ throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+ } catch (IllegalArgumentException e) {
+ throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+ } catch (InstantiationException e) {
+ throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+ } catch (IllegalAccessException e) {
+ throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+ } catch (InvocationTargetException e) {
+ throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+ }
+
+ throw createException("Class " + className + " has no public single argument constructor!");
}
- public String[] getStopwords() {
- return stopwords;
+ private FilterBuildingException createException(String message, Throwable cause){
+ String extendedMessage = message + " at line " + currentLocateable.getLineNumber();
+ return new FilterBuildingException(extendedMessage, cause);
}
- public Map<String, List<String>> getHypernyms() {
- return hypernyms;
+private FilterBuildingException createException(String message){
+ String extendedMessage = message + " at line " + currentLocateable.getLineNumber();
+ return new FilterBuildingException(extendedMessage);
}
- public Map<String, Map<String, String>> getMappings() {
- return mappings;
+Map<String, TokenFilterFactory> getCachedFactories() {
+ return cachedFactories;
}
+
}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+public class FilterBuildingException extends Exception {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public FilterBuildingException(String message) {
+ super(message);
+ }
+
+ public FilterBuildingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public FilterBuildingException(Throwable cause) {
+ super(cause);
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+import org.apache.lucene.index.IndexWriter;
+
+public interface IndexWriterProvider {
+
+ public IndexWriter getIndexWriter();
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.uima.resource.DataResource;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.SharedResourceObject;
+
+public class IndexWriterProviderImpl implements IndexWriterProvider, SharedResourceObject{
+
+ public static final String USE_COMPOUND_FILE_FORMAT_PROPERTY = "useCompoundFileFormat";
+ public static final String RAMBUFFER_SIZE_PROPERTY = "RAMBufferSize";
+ public static final String INDEX_PATH_PROPERTY = "indexPath";
+ public static final String MAX_FIELD_LENGTH_PROPERTY = "maxFieldLength";
+ private static final String UNIQUE_INDEX_PROPERTY = "uniqueIndex";
+ public IndexWriter indexWriter;
+
+ public IndexWriter getIndexWriter() {
+ return indexWriter;
+ }
+
+ public void load(DataResource dataResource) throws ResourceInitializationException {
+ Properties properties = loadProperties(dataResource);
+
+ String indexPath = getIndexPath(properties);
+ if (getUniqueIndexOrDefault(properties))
+ indexPath = createUniqueIndexPath(indexPath);
+
+ MaxFieldLength maxFieldLength = getMaxFieldLengthOrDefault(properties);
+ createIndexWriter(indexPath, maxFieldLength);
+
+ Double ramBufferSize = getRAMBufferSizeOrDefault(properties);
+ Boolean useCompoundFileFormat = getUseCompoundFormatOrDefault(properties);
+ configureIndexWriter(ramBufferSize, useCompoundFileFormat);
+ }
+
+ private Properties loadProperties(DataResource dataResource)
+ throws ResourceInitializationException {
+
+ Properties properties = new Properties();
+ try {
+ properties.load(dataResource.getInputStream());
+ } catch (IOException e) {
+ throw new ResourceInitializationException(e);
+ }
+
+ return properties;
+ }
+
+ private Boolean getUseCompoundFormatOrDefault(Properties properties) {
+ String useCompoundFileFormatAsString = properties.getProperty(USE_COMPOUND_FILE_FORMAT_PROPERTY);
+ Boolean useCompoundFileFormat = true;
+ if( useCompoundFileFormatAsString != null )
+ useCompoundFileFormat = new Boolean(useCompoundFileFormatAsString);
+
+ return useCompoundFileFormat;
+ }
+
+ private String getIndexPath(Properties properties){
+ return properties.getProperty(INDEX_PATH_PROPERTY);
+ }
+
+ private String createUniqueIndexPath(String indexPath){
+ String uniqueIndexPath = indexPath + "-" + getHostName() + "-" + getPID();
+ return uniqueIndexPath;
+ }
+
+ private Boolean getUniqueIndexOrDefault(Properties properties) {
+ String uniqueIndexAsString = properties.getProperty(UNIQUE_INDEX_PROPERTY);
+ Boolean uniqueIndex = false;
+ if( uniqueIndexAsString != null )
+ uniqueIndex = new Boolean(uniqueIndexAsString);
+
+ return uniqueIndex;
+ }
+
+ private Double getRAMBufferSizeOrDefault(Properties properties) {
+ String ramBufferSizeAsString = properties.getProperty(RAMBUFFER_SIZE_PROPERTY);
+ Double ramBufferSize = IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB;
+ if( ramBufferSizeAsString != null )
+ ramBufferSize = new Double(ramBufferSizeAsString);
+ return ramBufferSize;
+ }
+
+ private MaxFieldLength getMaxFieldLengthOrDefault(Properties properties) {
+ String maxFieldLengthAsString = properties.getProperty(MAX_FIELD_LENGTH_PROPERTY);
+ if( maxFieldLengthAsString == null )
+ return IndexWriter.MaxFieldLength.UNLIMITED;
+ else{
+ Integer maxFieldLength = new Integer(maxFieldLengthAsString);
+ return new MaxFieldLength(maxFieldLength);
+ }
+ }
+
+ private void createIndexWriter(String indexPath, MaxFieldLength maxFieldLength) throws ResourceInitializationException {
+ try {
+ indexWriter = new IndexWriter(indexPath, new StandardAnalyzer(), true, maxFieldLength);
+ } catch (CorruptIndexException e) {
+ throw new ResourceInitializationException(e);
+ } catch (LockObtainFailedException e) {
+ throw new ResourceInitializationException(e);
+ } catch (IOException e) {
+ throw new ResourceInitializationException(e); }
+ }
+
+ private void configureIndexWriter(Double ramBufferSize, Boolean useCompoundFileFormat) {
+ indexWriter.setRAMBufferSizeMB(ramBufferSize);
+ indexWriter.setUseCompoundFile(true);
+ }
+
+ private String getPID() {
+ String id = ManagementFactory.getRuntimeMXBean().getName();
+ return id.substring(0, id.indexOf("@"));
+ }
+
+ private String getHostName() {
+ InetAddress address;
+ String hostName;
+ try {
+ address = InetAddress.getLocalHost();
+ hostName = address.getHostName();
+ } catch (UnknownHostException e) {
+ throw new IllegalStateException(e);
+ }
+ return hostName;
+ }
+
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java Fri Jul 24 08:58:52 2009
@@ -25,6 +25,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.uima.lucas.indexer.mapping.AnnotationDescription;
import org.apache.uima.lucas.indexer.util.TokenStreamStringConcatenator;
public class Tokenizer {
@@ -49,8 +50,8 @@
throws IOException {
String tokenizer = description.getTokenizer();
if (tokenizer != null && !tokenizer.equals(TOKENIZER_CAS)) {
- String tokenStreamAsString =
- tokenStreamStringConcatenator.tokenStreamToStringWithDelimiter(tokenStream, " ");
+ String tokenStreamAsString = tokenStreamStringConcatenator.tokenStreamToStringWithDelimiter(
+ tokenStream, " ");
StringReader stringReader = new StringReader(tokenStreamAsString);
if (tokenizer.equals(TOKENIZER_WHITESPACE))
tokenStream = new WhitespaceTokenizer(stringReader);
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class AdditionFilter extends TokenFilter {
+
+ public static final Integer PREFIX = 0;
+
+ public static final Integer POSTFIX = 1;
+
+ private Integer position;
+
+ private String addition;
+
+ public AdditionFilter(TokenStream input, String addition, Integer position) {
+ super(input);
+ this.position = position;
+ this.addition = addition;
+ }
+
+ @Override
+ public Token next(Token aToken) throws IOException {
+ Token token = input.next(aToken);
+ if (token == null)
+ return null;
+
+ String tokenText = new String(token.termBuffer(), 0, token.termLength());
+ if (position.equals(PREFIX))
+ tokenText = addition.concat(tokenText);
+ else
+ tokenText = tokenText.concat(addition);
+
+ token.setTermBuffer(tokenText.toCharArray(), 0, tokenText.length());
+
+ return token;
+ }
+
+ public Integer getPosition() {
+ return position;
+ }
+
+ public String getAddition() {
+ return addition;
+ }
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class AdditionFilterFactory implements TokenFilterFactory {
+
+ public static final String POSTFIX_POSITION = "postfix";
+ public static final String PREFIX_POSITION = "prefix";
+
+ public TokenFilter createTokenFilter(TokenStream tokenStream,
+ Properties properties) throws IOException {
+
+ String postfix = properties.getProperty(POSTFIX_POSITION);
+ String addition = null;
+ Integer position = null;
+
+ if( postfix != null ){
+ position = AdditionFilter.POSTFIX;
+ addition = postfix;
+ }
+ else {
+ String prefix = properties.getProperty(PREFIX_POSITION);
+ if( prefix != null ){
+ position = AdditionFilter.PREFIX;
+ addition = prefix;
+ }
+ else
+ throw new IllegalArgumentException("no postfix or prefix attribute provided");
+ }
+
+ return new AdditionFilter(tokenStream, addition, position);
+ }
+
+ public void preloadResources(Properties properties) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java Fri Jul 24 08:58:52 2009
@@ -21,10 +21,8 @@
import java.io.IOException;
import java.text.Format;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -45,6 +43,8 @@
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
*
@@ -74,13 +74,11 @@
private Iterator<Annotation> annotationIterator; // iterates over annotations
private Iterator<FeatureStructure> featureStructureIterator; // iterates over feature structures
-
- // stored in feature arrays of an
- // annotation
+ // stored in feature arrays of an
+ // annotation
private Iterator<String> featureValueIterator; // iterates over the features of a feature
-
- // structure
+ // structure
private Annotation currentAnnotation;
@@ -108,23 +106,9 @@
* the type of the annotation
* @throws CASException
*/
- public AnnotationTokenStream(JCas cas, String sofaName, String typeName) throws CASException {
- super();
- jCas = cas.getView(sofaName);
- this.featureNames = Collections.EMPTY_LIST;
- this.featureFormats = Collections.EMPTY_MAP;
-
- try {
- annotationType = jCas.getTypeSystem().getType(typeName);
- logger.debug(typeName + ", found: " + (annotationType != null));
- logger.debug("featureNames: " + featureNames);
- initializeIterators();
- } catch (Exception e) {
- IllegalArgumentException exc =
- new IllegalArgumentException(e.getMessage() + " at type " + typeName);
- exc.initCause(e);
- throw exc;
- }
+ public AnnotationTokenStream(JCas cas, String sofaName, String typeName) throws InvalidTokenSourceException {
+ this(cas, sofaName, typeName, null, Collections.EMPTY_LIST, null,
+ Collections.EMPTY_MAP);
}
/**
@@ -144,30 +128,9 @@
*/
public AnnotationTokenStream(JCas cas, String sofaName, String typeName, String featureName,
- Format featureFormat) throws CASException {
- super();
- jCas = cas.getView(sofaName);
- this.featureNames = new ArrayList<String>();
- if (featureFormat != null) {
- featureFormats = new HashMap<String, Format>();
- featureFormats.put(featureName, featureFormat);
- } else
- this.featureFormats = Collections.EMPTY_MAP;
-
- featureNames.add(featureName);
-
- try {
- annotationType = jCas.getTypeSystem().getType(typeName);
- logger.debug(typeName + ", found: " + (annotationType != null));
- logger.debug("featureNames: " + featureNames);
- initializeIterators();
- } catch (Exception e) {
- IllegalArgumentException exc =
- new IllegalArgumentException(e.getMessage() + " at type " + typeName);
- exc.initCause(e);
- throw exc;
- }
-
+ Format featureFormat) throws InvalidTokenSourceException {
+ this(cas, sofaName, typeName, null, Lists.newArrayList(featureName), null,
+ Maps.immutableBiMap(featureName, featureFormat));
}
/**
@@ -192,28 +155,8 @@
*/
public AnnotationTokenStream(JCas cas, String sofaName, String typeName,
List<String> featureNames, String delimiter, Map<String, Format> featureFormats)
- throws CASException {
- super();
- jCas = cas.getView(sofaName);
- this.featureNames = featureNames;
- this.delimiter = delimiter;
-
- if (featureFormats == null)
- this.featureFormats = Collections.EMPTY_MAP;
- else
- this.featureFormats = featureFormats;
-
- try {
- annotationType = jCas.getTypeSystem().getType(typeName);
- logger.debug(typeName + ", found: " + (annotationType != null));
- logger.debug("featureNames: " + featureNames);
- initializeIterators();
- } catch (Exception e) {
- IllegalArgumentException exc =
- new IllegalArgumentException(e.getMessage() + " at type " + typeName);
- exc.initCause(e);
- throw exc;
- }
+ throws InvalidTokenSourceException {
+ this(cas, sofaName, typeName, null, featureNames, delimiter, featureFormats);
}
/**
@@ -233,26 +176,8 @@
* @throws CASException
*/
public AnnotationTokenStream(JCas cas, String sofaName, String typeName,
- List<String> featureNames, Map<String, Format> featureFormats) throws CASException {
- super();
- jCas = cas.getView(sofaName);
- this.featureNames = featureNames;
- if (featureFormats == null)
- this.featureFormats = Collections.EMPTY_MAP;
- else
- this.featureFormats = featureFormats;
-
- try {
- annotationType = jCas.getTypeSystem().getType(typeName);
- logger.debug(typeName + ", found: " + (annotationType != null));
- logger.debug("featureNames: " + featureNames);
- initializeIterators();
- } catch (Exception e) {
- IllegalArgumentException exc =
- new IllegalArgumentException(e.getMessage() + " at type " + typeName);
- exc.initCause(e);
- throw exc;
- }
+ List<String> featureNames, Map<String, Format> featureFormats) throws InvalidTokenSourceException {
+ this(cas, sofaName, typeName, null, featureNames, null, featureFormats);
}
/**
@@ -284,28 +209,8 @@
* @throws CASException
*/
public AnnotationTokenStream(JCas cas, String sofaName, String typeName, String featurePath,
- List<String> featureNames, Map<String, Format> featureFormats) throws CASException {
- super();
- jCas = cas.getView(sofaName);
- this.featurePath = featurePath;
- this.featureNames = featureNames;
- if (featureFormats == null)
- this.featureFormats = Collections.EMPTY_MAP;
- else
- this.featureFormats = featureFormats;
-
- try {
- annotationType = jCas.getTypeSystem().getType(typeName);
- logger.debug(typeName + ", found: " + (annotationType != null));
- logger.debug("featurePath: " + featurePath);
- logger.debug("featureNames: " + featureNames);
- initializeIterators();
- } catch (Exception e) {
- IllegalArgumentException exc =
- new IllegalArgumentException(e.getMessage() + " at type " + typeName);
- exc.initCause(e);
- throw exc;
- }
+ List<String> featureNames, Map<String, Format> featureFormats) throws InvalidTokenSourceException {
+ this(cas, sofaName, typeName, featurePath, featureNames, null, featureFormats);
}
/**
@@ -339,9 +244,9 @@
*/
public AnnotationTokenStream(JCas cas, String sofaName, String typeName, String featurePath,
List<String> featureNames, String delimiter, Map<String, Format> featureFormats)
- throws CASException {
+ throws InvalidTokenSourceException {
super();
- jCas = cas.getView(sofaName);
+
this.featurePath = featurePath;
this.featureNames = featureNames;
this.delimiter = delimiter;
@@ -349,20 +254,57 @@
this.featureFormats = Collections.EMPTY_MAP;
else
this.featureFormats = featureFormats;
+
+ getSofaCas(cas, sofaName);
+ getTypeForName(typeName);
+ validate(annotationType, featureNames, featurePath);
+
+ initializeIterators();
+ }
+
+ private void getTypeForName(String typeName) throws InvalidTokenSourceException{
+ annotationType = jCas.getTypeSystem().getType(typeName);
+ if( annotationType == null )
+ throw new InvalidTokenSourceException("Type " + typeName + " not found!");
+ }
+
+ private void getSofaCas(JCas cas, String sofaName) throws InvalidTokenSourceException {
try {
- annotationType = jCas.getTypeSystem().getType(typeName);
- logger.debug(typeName + ", found: " + (annotationType != null));
- logger.debug("featurePath: " + featurePath);
- logger.debug("featureNames: " + featureNames);
- initializeIterators();
- } catch (Exception e) {
- IllegalArgumentException exc =
- new IllegalArgumentException(e.getMessage() + " at type " + typeName);
- exc.initCause(e);
- throw exc;
+ jCas = cas.getView(sofaName);
+ } catch (CASException e) {
+ throw new InvalidTokenSourceException(e);
}
+ }
+ void validate(Type type, Collection<String> featureNames, String featurePath) throws InvalidTokenSourceException{
+ Type typeToValidate = findTypeWithPath(type, featurePath);
+
+ for( String featureName: featureNames ){
+ Feature feature = typeToValidate.getFeatureByBaseName(featureName);
+ if( feature == null )
+ throw new InvalidTokenSourceException("Type " + typeToValidate.getName() + " has no feature " + featureName + ". featurePath: " + featurePath);
+ }
+ }
+
+ private Type findTypeWithPath(Type type, String featurePath) throws InvalidTokenSourceException{
+ if( featurePath == null )
+ return type;
+
+ String[] featurePathElements = featurePath.split("\\.");
+ Type currentType = type;
+
+ for( String featurePathElement: featurePathElements ){
+ Feature feature = currentType.getFeatureByBaseName(featurePathElement);
+ if (feature == null)
+ throw new InvalidTokenSourceException("Type " + currentType.getName() + " has no feature " + featurePathElement);
+
+ currentType = feature.getRange();
+ if (currentType.isArray())
+ currentType = currentType.getComponentType();
+ }
+
+ return currentType;
}
@Override
@@ -376,8 +318,8 @@
featureStructureIterator = createFeatureStructureIterator(currentAnnotation, featurePath);
}
- featureValueIterator =
- createFeatureValueIterator(featureStructureIterator.next(), featureNames);
+ featureValueIterator = createFeatureValueIterator(featureStructureIterator.next(),
+ featureNames);
}
token.setStartOffset(currentAnnotation.getBegin());
@@ -389,10 +331,9 @@
} catch (Throwable e) {
- IOException ioException =
- new IOException(e + " at type " + annotationType.getName() + " features "
- + featureNames + " featurePath " + featurePath + " sofa "
- + jCas.getViewName(), e);
+ IOException ioException = new IOException(e + " at type " + annotationType.getName()
+ + " features " + featureNames + " featurePath " + featurePath + " sofa "
+ + jCas.getViewName(), e);
logger.error(ioException);
throw ioException;
}
@@ -409,9 +350,8 @@
}
protected void initializeIterators() {
- annotationIterator =
- Iterators.filter(jCas.getAnnotationIndex(annotationType).iterator(),
- new NotNullPredicate<Annotation>());
+ annotationIterator = Iterators.filter(jCas.getAnnotationIndex(annotationType).iterator(),
+ new NotNullPredicate<Annotation>());
if (!annotationIterator.hasNext()) {
featureStructureIterator = Iterators.emptyIterator();
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class ConcatFilter extends TokenFilter{
+
+ private int startOffset;
+ private String concatString;
+ private Token lastToken;
+
+ protected ConcatFilter(TokenStream input, String concatString) {
+ super(input);
+ this.concatString = concatString;
+ startOffset = -1;
+ }
+
+ @Override
+ public Token next(Token reusableToken) throws IOException {
+ String tokenText = concatenatedInputTokenTerms(reusableToken);
+ reusableToken.setTermBuffer(tokenText.toCharArray(), 0, tokenText.length());
+ reusableToken.setStartOffset(startOffset);
+ reusableToken.setEndOffset(lastToken.endOffset());
+ return reusableToken;
+ }
+
+ private String concatenatedInputTokenTerms(Token reusableToken) throws IOException {
+ StringBuffer stringBuffer = new StringBuffer();
+ Token nextToken = input.next(reusableToken);
+ lastToken = null;
+
+ if( startOffset == -1 )
+ startOffset = nextToken.startOffset();
+
+ while( nextToken != null ){
+ lastToken = nextToken;
+ stringBuffer.append(nextToken.term());
+ nextToken = input.next(reusableToken);
+ if( nextToken != null )
+ stringBuffer.append(concatString);
+ }
+
+ return stringBuffer.toString();
+ }
+
+ protected String getConcatString() {
+ return concatString;
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class ConcatFilterFactory implements TokenFilterFactory {
+
+ public static final String CONCAT_STRING_PARAMETER = "concatString";
+
+ public TokenFilter createTokenFilter(TokenStream tokenStream,
+ Properties properties) throws IOException {
+
+ String concatString = properties.getProperty(CONCAT_STRING_PARAMETER);
+ return new ConcatFilter(tokenStream, concatString);
+ }
+
+ public void preloadResources(Properties properties) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+}