You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by jo...@apache.org on 2009/07/24 10:58:55 UTC

svn commit: r797373 [2/8] - in /incubator/uima/sandbox/trunk/Lucas: ./ docbook/ docbook/LuceneCASConsumerUserGuide/ docbook/LuceneCASConsumerUserGuide/images/ src/main/java/org/apache/uima/lucas/consumer/ src/main/java/org/apache/uima/lucas/indexer/ sr...

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/consumer/LuceneCASIndexer.java Fri Jul 24 08:58:52 2009
@@ -19,281 +19,259 @@
 
 package org.apache.uima.lucas.consumer;
 
-import java.io.BufferedReader;
-import java.io.FileReader;
+import java.io.File;
 import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
 import org.apache.log4j.Logger;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.uima.UimaContext;
 import org.apache.uima.cas.CAS;
 import org.apache.uima.cas.CASException;
 import org.apache.uima.collection.CasConsumer_ImplBase;
 import org.apache.uima.jcas.JCas;
-import org.apache.uima.lucas.indexer.AnnotationDescription;
 import org.apache.uima.lucas.indexer.AnnotationTokenStreamBuilder;
+import org.apache.uima.lucas.indexer.AnnotationTokenStreamBuildingException;
 import org.apache.uima.lucas.indexer.DocumentBuilder;
 import org.apache.uima.lucas.indexer.FieldBuilder;
-import org.apache.uima.lucas.indexer.FieldDescription;
+import org.apache.uima.lucas.indexer.FieldBuildingException;
 import org.apache.uima.lucas.indexer.FilterBuilder;
-import org.apache.uima.lucas.indexer.IndexMappingFileReader;
+import org.apache.uima.lucas.indexer.FilterBuildingException;
+import org.apache.uima.lucas.indexer.IndexWriterProvider;
 import org.apache.uima.lucas.indexer.Tokenizer;
-import org.apache.uima.lucas.indexer.util.MapFileReader;
-import org.apache.uima.lucas.indexer.util.MultimapFileReader;
-import org.apache.uima.lucas.indexer.util.PlainFileReader;
+import org.apache.uima.lucas.indexer.analysis.DefaultFilterFactoryRegistry;
+import org.apache.uima.lucas.indexer.analysis.TokenFilterFactory;
+import org.apache.uima.lucas.indexer.mapping.AnnotationDescription;
+import org.apache.uima.lucas.indexer.mapping.AnnotationMapper;
+import org.apache.uima.lucas.indexer.mapping.ElementMapper;
+import org.apache.uima.lucas.indexer.mapping.FeatureMapper;
+import org.apache.uima.lucas.indexer.mapping.FieldDescription;
+import org.apache.uima.lucas.indexer.mapping.FieldMapper;
+import org.apache.uima.lucas.indexer.mapping.FilterDescription;
+import org.apache.uima.lucas.indexer.mapping.FilterMapper;
+import org.apache.uima.lucas.indexer.mapping.MappingFileReader;
+import org.apache.uima.resource.ResourceAccessException;
 import org.apache.uima.resource.ResourceInitializationException;
 import org.apache.uima.resource.ResourceProcessException;
+import org.xml.sax.SAXException;
 
 /**
- * Reads CAS object and writes the particular information in fields of a Lucene index. requires a
- * mapping file
+ * Reads CAS object and writes the particular information in fields of a Lucene index.
+ * requires a mapping file
  */
 public class LuceneCASIndexer extends CasConsumer_ImplBase {
 
-  private static final Logger logger = Logger.getLogger(LuceneCASIndexer.class);
+	private static final String RESOURCE_INDEX_WRITER_PROVIDER = "indexWriterProvider";
+
+	private static final Logger logger = Logger
+			.getLogger(LuceneCASIndexer.class);
 
-  public final static String PARAM_OUTDIR = "indexOutDir";
+	public final static String PARAM_MAPPINGFILE = "mappingFile";
 
-  public final static String PARAM_MAPPINGFILE = "mappingFile";
+	private IndexWriter indexWriter;
 
-  public final static String PARAM_STOPWORDFILE = "stopwordFile";
+	private Collection<FieldDescription> fieldDescriptions;
+
+	private DocumentBuilder documentBuilder;
+
+	private FieldBuilder fieldBuilder;
+
+	private FilterBuilder filterBuilder;
+
+	private AnnotationTokenStreamBuilder annotationTokenStreamBuilder;
+
+	private Tokenizer tokenizer;
+
+	private Map<String, TokenFilterFactory> defaultFilterFactoryRegistry;
+
+	/**
+	 * initializes the analyzer
+	 */
+	public void initialize() throws ResourceInitializationException {
+		createFieldDescriptions();
+		getIndexWriterInstance();
+		createFilterBuilderWithPreloadedResources();
+
+		annotationTokenStreamBuilder = new AnnotationTokenStreamBuilder();
+		tokenizer = new Tokenizer();
+		fieldBuilder = new FieldBuilder(filterBuilder);
+		documentBuilder = new DocumentBuilder();
+	}
+
+	private void createFieldDescriptions()
+			throws ResourceInitializationException {
+		String mappingFilePath = (String) getConfigParameterValue(PARAM_MAPPINGFILE);
+
+		try {
+			MappingFileReader indexMappingFileReader = createMappingFileReader();
+			File mappingFile = new File(mappingFilePath);
+			fieldDescriptions = indexMappingFileReader
+					.readFieldDescriptionsFromFile(mappingFile);
+		} catch (IOException e) {
+			throw new ResourceInitializationException(e);
+		}
+	}
+
+	private MappingFileReader createMappingFileReader() throws IOException{
+		try {
+			SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+			Map<String, ElementMapper<?>> elementMappers = new HashMap<String, ElementMapper<?>>();
+			elementMappers.put(MappingFileReader.ANNOTATION, new AnnotationMapper());
+			elementMappers.put(MappingFileReader.FILTER, new FilterMapper());
+			elementMappers.put(MappingFileReader.FIELD, new FieldMapper());
+			elementMappers.put(MappingFileReader.FEATURE, new FeatureMapper());			
+			return new MappingFileReader(parser, elementMappers);
+			
+		} catch (ParserConfigurationException e) {
+			throw new IOException("Can't build SAXParser", e);
+		} catch (SAXException e) {
+			throw new IOException("Can't build SAXParser", e);
+		}
+	}
+
+	private void getIndexWriterInstance()
+			throws ResourceInitializationException {
+		UimaContext uimaContext = getUimaContext();
+		IndexWriterProvider indexWriterProvider;
+		try {
+			indexWriterProvider = (IndexWriterProvider) uimaContext
+					.getResourceObject(RESOURCE_INDEX_WRITER_PROVIDER);
+		} catch (ResourceAccessException e) {
+			throw new ResourceInitializationException(e);
+		}
+		indexWriter = indexWriterProvider.getIndexWriter();
+	}
+
+	private void createFilterBuilderWithPreloadedResources()
+			throws ResourceInitializationException {
+		defaultFilterFactoryRegistry = new DefaultFilterFactoryRegistry()
+				.getDefaultRegistry();
+		try {
+			preloadResources(fieldDescriptions, defaultFilterFactoryRegistry);
+		} catch (IOException e) {
+			throw new ResourceInitializationException(e);
+		}
+		filterBuilder = new FilterBuilder(defaultFilterFactoryRegistry);
+	}
+
+	void preloadResources(Collection<FieldDescription> fieldDescriptions,
+			Map<String, TokenFilterFactory> defaultFilterFactoryRegistry)
+			throws IOException {
+
+		for (FieldDescription fieldDescription : fieldDescriptions) {
+			for (FilterDescription filterDescription : fieldDescription
+					.getFilterDescriptions()) {
+				TokenFilterFactory tokenFilterFactory = defaultFilterFactoryRegistry
+						.get(filterDescription.getName());
+				if (tokenFilterFactory != null)
+					tokenFilterFactory.preloadResources(filterDescription
+							.getProperties());
+			}
+
+			for (AnnotationDescription annotationDescription : fieldDescription
+					.getAnnotationDescriptions()) {
+				for (FilterDescription filterDescription : annotationDescription
+						.getFilterDescriptions()) {
+					TokenFilterFactory tokenFilterFactory = defaultFilterFactoryRegistry
+							.get(filterDescription.getName());
+					if (tokenFilterFactory != null)
+						tokenFilterFactory.preloadResources(filterDescription
+								.getProperties());
+				}
+			}
+		}
+	}
+
+	public void processCas(CAS cas) throws ResourceProcessException {
+
+		try {
+			JCas jCas = cas.getJCas();
+			Collection<Field> fields = new ArrayList<Field>();
+			// iterate over field descriptions from mapping file
+			for (FieldDescription fieldDescription : fieldDescriptions) {
+				Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
+				// iterate over annotation descriptions
+				for (AnnotationDescription annotationDescription : fieldDescription
+						.getAnnotationDescriptions()) {
+					// create annotation token stream
+					TokenStream tokenStream = annotationTokenStreamBuilder
+							.createAnnotationTokenStream(jCas,
+									annotationDescription);
+
+					// needs (re)tokenization ?
+					if (tokenizer.needsTokenization(annotationDescription))
+						tokenStream = tokenizer.tokenize(tokenStream,
+								annotationDescription);
+
+					// wrap with filters
+					tokenStream = filterBuilder.filter(tokenStream,
+							annotationDescription.getFilterDescriptions());
+					tokenStreams.add(tokenStream);
+				}
+
+				// create fields
+				fields.addAll(fieldBuilder.createFields(tokenStreams,
+						fieldDescription));
+			}
+			// create document and add to index
+			Document document = documentBuilder.createDocument(fields);
+			indexWriter.addDocument(document);
+
+		} catch (AnnotationTokenStreamBuildingException e) {
+			logger.error("processCas(CAS)", e);
+			throw new ResourceProcessException(e);
+		} catch (IOException e) {
+			logger.error("processCas(CAS)", e);
+			throw new ResourceProcessException(e);
+		} catch (FieldBuildingException e) {
+			logger.error("processCas(CAS)", e);
+			throw new ResourceProcessException(e);
+		} catch (FilterBuildingException e) {
+			logger.error("processCas(CAS)", e);
+			throw new ResourceProcessException(e);
+		} catch (CASException e) {
+			logger.error("processCas(CAS)", e);
+			throw new ResourceProcessException(e);
+		}
+	}
+
+	@Override
+	public void destroy() {
+		logger.info("destroy " + LuceneCASIndexer.class);
+		optimizeIndex();
+		super.destroy();
+	}
+
+	public void optimizeIndex() {
+		try {
+			logger.info("optimizing the index now!");
+			indexWriter.optimize();
+			indexWriter.close();
+		} catch (IOException e) {
+			logger.error("exception while closing index", e);
+		}
+	}
+
+	FilterBuilder getFilterBuilder() {
+		return filterBuilder;
+	}
+
+	Collection<FieldDescription> getFieldDescriptions() {
+		return fieldDescriptions;
+	}
+
+	IndexWriter getIndexWriter() {
+		return indexWriter;
+	}
 
-  public final static String PARAM_HYPERNYMFILE = "hypernymFile";
-
-  public final static String PARAM_TOKEN_MAPPINGFILE = "tokenMappingFile";
-
-  public final static String PARAM_UNIQUE_INDEX = "uniqueIndex";
-
-  public final static String PARAM_RAM_BUFFER_SIZE = "ramBufferSize";
-
-  public final static String PARAM_COMPOUND_FILE_FORMAT = "compoundFileFormat";
-
-  public final static String PARAM_HEADER_TYPE_NAME = "headerTypeName";
-
-  public final static String PARAM_DOC_ID_FEATURE_NAME = "docIdFeatureName";
-
-  private IndexWriter indexWriter;
-
-  private Collection<FieldDescription> fieldDescriptions;
-
-  private DocumentBuilder documentBuilder;
-
-  private FieldBuilder fieldBuilder;
-
-  private FilterBuilder filterBuilder;
-
-  private AnnotationTokenStreamBuilder annotationTokenStreamBuilder;
-
-  private Tokenizer tokenizer;
-
-  // private String headerTypeName;
-  // private String docIdFeatureName;
-
-  /**
-   * initializes the analyzer
-   */
-  public void initialize() throws ResourceInitializationException {
-
-    String mappingFile = (String) getConfigParameterValue(PARAM_MAPPINGFILE);
-    String stopwordFile = (String) getConfigParameterValue(PARAM_STOPWORDFILE);
-    String hypernymFile = (String) getConfigParameterValue(PARAM_HYPERNYMFILE);
-    String[] mappingFiles = (String[]) getConfigParameterValue(PARAM_TOKEN_MAPPINGFILE);
-    Boolean uniqueIndex = (Boolean) getConfigParameterValue(PARAM_UNIQUE_INDEX);
-    String outDir = (String) getConfigParameterValue(PARAM_OUTDIR);
-
-    // headerTypeName = (String) getConfigParameterValue(PARAM_HEADER_TYPE_NAME);
-    // docIdFeatureName = (String) getConfigParameterValue(PARAM_DOC_ID_FEATURE_NAME);
-
-    Integer ramBufferSize = (Integer) getConfigParameterValue(PARAM_RAM_BUFFER_SIZE);
-    Boolean compoundFileFormat = (Boolean) getConfigParameterValue(PARAM_COMPOUND_FILE_FORMAT);
-
-    try {
-      if (uniqueIndex)
-        outDir += "-" + getHostName() + "-" + getPID();
-
-      IndexMappingFileReader indexMappingFileReader = new IndexMappingFileReader();
-      fieldDescriptions = indexMappingFileReader.readFieldDescriptionsFromFile(mappingFile);
-
-      indexWriter = new IndexWriter(outDir, new StandardAnalyzer());
-      if (ramBufferSize != null)
-        indexWriter.setRAMBufferSizeMB(ramBufferSize);
-      if (compoundFileFormat != null)
-        indexWriter.setUseCompoundFile(compoundFileFormat);
-
-      String[] stopwords = null;
-      Map<String, Map<String, String>> mappings = null;
-      Map<String, List<String>> hypernyms = null;
-
-      if (stopwordFile != null) {
-        PlainFileReader stopwordReader =
-                new PlainFileReader(new BufferedReader(new FileReader(stopwordFile)));
-        stopwords = stopwordReader.readLines();
-      } else
-        stopwords = new String[0];
-
-      if (hypernymFile != null) {
-        MultimapFileReader hypernymReader =
-                new MultimapFileReader(new BufferedReader(new FileReader(hypernymFile)));
-        hypernyms = hypernymReader.readMultimap();
-      } else
-        hypernyms = Collections.EMPTY_MAP;
-
-      if (mappingFiles != null && mappingFiles.length != 0)
-        mappings = readMappingFiles(mappingFiles);
-      else
-        mappings = Collections.EMPTY_MAP;
-
-      annotationTokenStreamBuilder = new AnnotationTokenStreamBuilder();
-      tokenizer = new Tokenizer();
-      filterBuilder = new FilterBuilder(stopwords, hypernyms, mappings);
-      fieldBuilder = new FieldBuilder();
-      documentBuilder = new DocumentBuilder();
-
-    } catch (Throwable e) {
-      logger.error("initialize()", e);
-      throw new ResourceInitializationException(e);
-    }
-  }
-
-  private Map<String, Map<String, String>> readMappingFiles(String[] mappingFiles)
-          throws IOException {
-    Map<String, Map<String, String>> mappings = new HashMap<String, Map<String, String>>();
-    for (String mappingFile : mappingFiles) {
-      MapFileReader reader = new MapFileReader(new BufferedReader(new FileReader(mappingFile)));
-      Map<String, String> mapping = reader.readMap();
-
-      if (mappingFile.lastIndexOf("/") != -1)
-        mappingFile = mappingFile.substring(mappingFile.lastIndexOf("/") + 1);
-
-      mappings.put(mappingFile, mapping);
-      reader.close();
-    }
-
-    return mappings;
-  }
-
-  /**
-   * optimizes the index
-   * 
-   */
-  public void optimizeIndex() {
-    try {
-      logger.info("optimizing the index now!");
-      indexWriter.optimize();
-      indexWriter.close();
-    } catch (IOException e) {
-      e.printStackTrace();
-    }
-  }
-
-  @Override
-  public void destroy() {
-    logger.info("destroy " + LuceneCASIndexer.class);
-    optimizeIndex();
-    super.destroy();
-  }
-
-  /**
-   * processes the CAS
-   */
-  public void processCas(CAS cas) throws ResourceProcessException {
-    JCas jCas = null;
-    try {
-      jCas = cas.getJCas();
-    } catch (CASException e) {
-      logger.error("processCas(CAS)", e);
-    }
-
-    /*
-     * if( headerTypeName != null && docIdFeatureName != null ){ String docId = getDocID(jCas,
-     * headerTypeName, docIdFeatureName); logger.info("Processing document "+ docId ); }
-     */
-
-    try {
-      Collection<Field> fields = new ArrayList<Field>();
-      // iterate over field descriptions from mapping file
-      for (FieldDescription fieldDescription : fieldDescriptions) {
-        Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
-        // iterate over annotation descriptions
-        for (AnnotationDescription annotationDescription : fieldDescription
-                .getAnnotationDescriptions()) {
-          // create annotation token stream
-          TokenStream tokenStream =
-                  annotationTokenStreamBuilder.createAnnotationTokenStream(jCas,
-                          annotationDescription);
-
-          // needs (re)tokenization ?
-          if (tokenizer.needsTokenization(annotationDescription))
-            tokenStream = tokenizer.tokenize(tokenStream, annotationDescription);
-
-          // wrap with filters
-          tokenStream = filterBuilder.filter(tokenStream, annotationDescription);
-          tokenStreams.add(tokenStream);
-        }
-
-        // create fields
-        fields.addAll(fieldBuilder.createFields(tokenStreams, fieldDescription));
-      }
-      // create document and add to index
-      Document document = documentBuilder.createDocument(fields);
-      indexWriter.addDocument(document);
-
-    } catch (Throwable e) {
-      logger.error("processCas(CAS)", e); //$NON-NLS-1$
-      throw new ResourceProcessException(e);
-    }
-  }
-
-  protected FilterBuilder getFilterBuilder() {
-    return filterBuilder;
-  }
-
-  protected Collection<FieldDescription> getFieldDescriptions() {
-    return fieldDescriptions;
-  }
-
-  protected IndexWriter getIndexWriter() {
-    return indexWriter;
-  }
-
-  protected String getPID() {
-    String id = ManagementFactory.getRuntimeMXBean().getName();
-    return id.substring(0, id.indexOf("@"));
-  }
-
-  public String getHostName() {
-    InetAddress address;
-    String hostName;
-    try {
-      address = InetAddress.getLocalHost();
-      hostName = address.getHostName();
-    } catch (UnknownHostException e) {
-      throw new IllegalStateException(e);
-    }
-    return hostName;
-  }
-
-  /*
-   * public String getDocID(JCas cas, String headerTypeName, String docIdFeatureName){
-   * 
-   * Type headerType = cas.getTypeSystem().getType(headerTypeName); if( headerType != null ){
-   * FSIterator headerIterator = cas.getAnnotationIndex(headerType).iterator(); if(
-   * headerIterator.hasNext() ){ FeatureStructure headerFeatureStructure = headerIterator.get();
-   * Feature docIdFeature = headerType.getFeatureByBaseName(docIdFeatureName); return
-   * headerFeatureStructure.getFeatureValueAsString(docIdFeature); } }
-   * 
-   * return null; }
-   */
-}
+}
\ No newline at end of file

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuilder.java Fri Jul 24 08:58:52 2009
@@ -22,22 +22,29 @@
 import java.text.DecimalFormat;
 import java.text.Format;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.log4j.Logger;
 import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.CASException;
 import org.apache.uima.jcas.JCas;
 import org.apache.uima.lucas.indexer.analysis.AnnotationTokenStream;
+import org.apache.uima.lucas.indexer.analysis.InvalidTokenSourceException;
+import org.apache.uima.lucas.indexer.mapping.AnnotationDescription;
+import org.apache.uima.lucas.indexer.mapping.FeatureDescription;
 
 /**
  * Builds annotation token streams wrapped with filters according to annotation descriptions.
  */
 public class AnnotationTokenStreamBuilder {
 
+  private static final Logger LOGGER = Logger.getLogger(AnnotationTokenStreamBuilder.class);
+
+
   public AnnotationTokenStream createAnnotationTokenStream(JCas jCas,
-          AnnotationDescription annotationDescription) throws CASException {
+          AnnotationDescription annotationDescription) throws AnnotationTokenStreamBuildingException {
 
     String sofaName = annotationDescription.getSofa();
     sofaName = sofaName == null ? CAS.NAME_DEFAULT_SOFA : sofaName;
@@ -47,33 +54,64 @@
     AnnotationTokenStream annotationTokenStream = null;
 
     String typeName = annotationDescription.getType();
-    if (annotationDescription.getFeatureDescriptions().size() == 0) // use coveredText
-      annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName);
-    else {
+    if (annotationDescription.getFeatureDescriptions().size() == 0)
+		try {
+			annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName);
+		} catch (InvalidTokenSourceException e) {
+			throw createException(annotationDescription, e); 
+		}
+	else {
       String featurePath = annotationDescription.getFeaturePath();
-      String delimiter = annotationDescription.getConcatString();
+      String delimiter = annotationDescription.getFeatureValueDelimiterString();
+
+      Collection<FeatureDescription> featureDescriptions = annotationDescription.getFeatureDescriptions();
+      List<String> featureNames = extractFeatureNames(featureDescriptions);
+      Map<String, Format> featureFormats = extractFeatureFormats(featureDescriptions);
 
+      if (featurePath != null)
+		try {
+			annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName, featurePath,
+			        featureNames, delimiter, featureFormats);
+		} catch (InvalidTokenSourceException e) {
+			throw createException(annotationDescription, e);
+		}
+	else
+		try {
+			annotationTokenStream = new AnnotationTokenStream(jCas, sofaName, typeName, featureNames,
+			        delimiter, featureFormats);
+		} catch (InvalidTokenSourceException e) {
+			throw createException(annotationDescription, e);
+		}
+
+    }
+
+    return annotationTokenStream;
+  }
+
+  private List<String> extractFeatureNames(Collection<FeatureDescription> featureDescriptions){
       List<String> featureNames = new ArrayList<String>();
-      Map<String, Format> featureFormats = new HashMap<String, Format>();
-      for (FeatureDescription featureDescription : annotationDescription.getFeatureDescriptions()) {
+      for (FeatureDescription featureDescription : featureDescriptions) {
         featureNames.add(featureDescription.getFeatureName());
+      }
+      return featureNames;
+  }
+  
+  private Map<String, Format> extractFeatureFormats(Collection<FeatureDescription> featureDescriptions){
+      Map<String, Format> featureFormats = new HashMap<String, Format>();
+      for (FeatureDescription featureDescription : featureDescriptions) {
         if (featureDescription.getNumberFormat() != null) {
           Format format = new DecimalFormat(featureDescription.getNumberFormat());
           featureFormats.put(featureDescription.getFeatureName(), format);
         }
       }
-
-      if (featurePath != null)
-        annotationTokenStream =
-                new AnnotationTokenStream(jCas, sofaName, typeName, featurePath, featureNames,
-                        delimiter, featureFormats);
-      else
-        annotationTokenStream =
-                new AnnotationTokenStream(jCas, sofaName, typeName, featureNames, delimiter,
-                        featureFormats);
-
-    }
-
-    return annotationTokenStream;
+	  return featureFormats;
   }
+  
+  private AnnotationTokenStreamBuildingException createException(AnnotationDescription annotationDescription,
+		  InvalidTokenSourceException e) {
+	  String message = "Can't build TokenStream from annotation at line " +
+	  annotationDescription.getLineNumber();
+	  return new AnnotationTokenStreamBuildingException(message, e);
+  }
+
 }

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/AnnotationTokenStreamBuildingException.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+public class AnnotationTokenStreamBuildingException extends Exception {
+
+	public AnnotationTokenStreamBuildingException(String message){
+		super(message);
+	}
+	
+	public AnnotationTokenStreamBuildingException(String message, Throwable cause){
+		super(message, cause);
+	}
+	
+	public AnnotationTokenStreamBuildingException(Throwable cause){
+		super(cause);
+	}
+
+}

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/DocumentBuilder.java Fri Jul 24 08:58:52 2009
@@ -19,16 +19,18 @@
 
 package org.apache.uima.lucas.indexer;
 
-import java.io.IOException;
 import java.util.Collection;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.uima.cas.CASException;
 
 public class DocumentBuilder {
 
-  public Document createDocument(Collection<Field> fields) throws CASException, IOException {
+  public DocumentBuilder() {
+    super();
+  }
+
+  public Document createDocument(Collection<Field> fields) {
     Document document = new Document();
 
     for (Field field : fields)

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuilder.java Fri Jul 24 08:58:52 2009
@@ -29,147 +29,205 @@
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field.TermVector;
-import org.apache.uima.cas.CASException;
 import org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator;
 import org.apache.uima.lucas.indexer.analysis.TokenStreamMerger;
+import org.apache.uima.lucas.indexer.mapping.FieldDescription;
+import org.apache.uima.lucas.indexer.mapping.FilterDescription;
 import org.apache.uima.lucas.indexer.util.TokenStreamStringConcatenator;
 
 public class FieldBuilder {
 
-  public static final String FIELD_INDEX_NO = "no";
+	public static final String FIELD_INDEX_NO = "no";
 
-  public static final String FIELD_INDEX_YES = "yes";
+	public static final String FIELD_INDEX_YES = "yes";
 
-  public static final String FIELD_INDEX_NO_NORMS = "no_norms";
+	public static final String FIELD_INDEX_NO_NORMS = "no_norms";
 
-  public static final String FIELD_INDEX_NO_TF = "no_tf";
+	public static final String FIELD_INDEX_NO_TF = "no_tf";
 
-  public static final String FIELD_INDEX_NO_NORMS_TF = "no_norms_tf";
-
-  public static final String FIELD_TERM_VECTOR_NO = "no";
-
-  public static final String FIELD_TERM_VECTOR_YES = "yes";
-
-  public static final String FIELD_TERM_VECTOR_WITH_OFFSETS = "offsets";
-
-  public static final String FIELD_TERM_VECTOR_WITH_POSITIONS = "positions";
-
-  public static final String FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS = "positions_offsets";
-
-  public static final String FIELD_STORE_NO = "no";
-
-  public static final String FIELD_STORE_YES = "yes";
-
-  public static final String FIELD_STORE_COMPRESS = "compress";
-
-  private TokenStreamStringConcatenator tokenStreamStringConcatenator;
-
-  public FieldBuilder() {
-    tokenStreamStringConcatenator = new TokenStreamStringConcatenator();
-  }
-
-  public Collection<Field> createFields(Collection<TokenStream> tokenStreams,
-          FieldDescription fieldDescription) throws IOException, CASException {
-
-    TokenStream tokenStream = null;
-    if (fieldDescription.getMerge())
-      tokenStream = new TokenStreamMerger(tokenStreams);
-    else if (tokenStreams.size() > 1)
-      tokenStream = new TokenStreamConcatenator(tokenStreams);
-    else if (tokenStreams.size() == 1)
-      tokenStream = tokenStreams.iterator().next();
-
-    String fieldName = fieldDescription.getName();
-    String delimiter = fieldDescription.getDelimiter();
-    Collection<Field> fields = new ArrayList<Field>();
-
-    Field.Store fieldStore = getFieldStore(fieldDescription.getStored());
-    Field.Index fieldIndex = getFieldIndex(fieldDescription.getIndex());
-    Field.TermVector fieldTermVector = getFieldTermVector(fieldDescription.getTermVector());
-
-    if (fieldStore == Field.Store.YES || fieldStore == Field.Store.COMPRESS)
-      fields.addAll(createStoredFields(fieldName, tokenStream, fieldStore, delimiter));
-
-    boolean omitTF =
-            fieldDescription.getIndex().equals(FIELD_INDEX_NO_TF)
-                    || fieldDescription.getIndex().equals(FIELD_INDEX_NO_NORMS_TF);
-
-    if (fieldIndex != Field.Index.NO)
-      fields.add(createIndexedField(fieldName, tokenStream, fieldIndex, fieldTermVector, omitTF));
-
-    return fields;
-  }
-
-  private Field createIndexedField(String fieldName, TokenStream tokenStream, Index fieldIndex,
-          TermVector fieldTermVector, boolean omitTF) {
-
-    Field field = new Field(fieldName, tokenStream, fieldTermVector);
-    if (fieldIndex == Field.Index.NOT_ANALYZED_NO_NORMS)
-      field.setOmitNorms(true);
-    field.setOmitTf(omitTF);
-
-    return field;
-  }
-
-  protected Collection<Field> createStoredFields(String fieldName, TokenStream tokenStream,
-          Store fieldStore, String delimiter) throws IOException {
-    Collection<Field> fields = new ArrayList<Field>();
-    if (delimiter != null) {
-      String value =
-              tokenStreamStringConcatenator
-                      .tokenStreamToStringWithDelimiter(tokenStream, delimiter);
-      fields.add(new Field(fieldName, value, fieldStore, Field.Index.NO));
-    } else {
-      Token nextToken = tokenStream.next(new Token());
-      while (nextToken != null) {
-        String value = new String(nextToken.termBuffer(), 0, nextToken.termLength());
-        fields.add(new Field(fieldName, value, fieldStore, Field.Index.NO));
-        nextToken = tokenStream.next(nextToken);
-      }
-    }
-    tokenStream.reset();
-
-    return fields;
-  }
-
-  protected Field.Index getFieldIndex(String index) {
-    if (index.equals(FIELD_INDEX_NO))
-      return Field.Index.NO;
-    else if (index.equals(FIELD_INDEX_YES))
-      return Field.Index.NOT_ANALYZED;
-    else if (index.equals(FIELD_INDEX_NO_NORMS))
-      return Field.Index.NOT_ANALYZED_NO_NORMS;
-    else if (index.equals(FIELD_INDEX_NO_NORMS_TF))
-      return Field.Index.NOT_ANALYZED_NO_NORMS;
-    else if (index.equals(FIELD_INDEX_NO_TF))
-      return Field.Index.NOT_ANALYZED;
-
-    throw new IllegalArgumentException("unknown index parameter: " + index);
-  }
-
-  protected Field.TermVector getFieldTermVector(String termVector) {
-    if (termVector.equals(FIELD_TERM_VECTOR_NO))
-      return Field.TermVector.NO;
-    else if (termVector.equals(FIELD_TERM_VECTOR_YES))
-      return Field.TermVector.YES;
-    else if (termVector.equals(FIELD_TERM_VECTOR_WITH_OFFSETS))
-      return Field.TermVector.WITH_OFFSETS;
-    else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS))
-      return Field.TermVector.WITH_POSITIONS;
-    else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS))
-      return Field.TermVector.WITH_POSITIONS_OFFSETS;
-
-    throw new IllegalArgumentException("unknown termVector parameter: " + termVector);
-  }
-
-  protected Field.Store getFieldStore(String store) {
-    if (store.equals(FIELD_STORE_NO))
-      return Field.Store.NO;
-    else if (store.equals(FIELD_STORE_YES))
-      return Field.Store.YES;
-    else if (store.equals(FIELD_STORE_COMPRESS))
-      return Field.Store.COMPRESS;
+	public static final String FIELD_INDEX_NO_NORMS_TF = "no_norms_tf";
+
+	public static final String FIELD_TERM_VECTOR_NO = "no";
+
+	public static final String FIELD_TERM_VECTOR_YES = "yes";
+
+	public static final String FIELD_TERM_VECTOR_WITH_OFFSETS = "offsets";
+
+	public static final String FIELD_TERM_VECTOR_WITH_POSITIONS = "positions";
+
+	public static final String FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS = "positions_offsets";
+
+	public static final String FIELD_STORE_NO = "no";
+
+	public static final String FIELD_STORE_YES = "yes";
+
+	public static final String FIELD_STORE_COMPRESS = "compress";
+
+	private TokenStreamStringConcatenator tokenStreamStringConcatenator;
+	private FilterBuilder filterBuilder;
+	private FieldDescription fieldDescription;
+	
+	public FieldBuilder(FilterBuilder filterBuilder) {
+		tokenStreamStringConcatenator = new TokenStreamStringConcatenator();
+		this.filterBuilder = filterBuilder;
+	}
+
+	public Collection<Field> createFields(Collection<TokenStream> tokenStreams,
+			FieldDescription fieldDescription) throws FieldBuildingException {
+
+		TokenStream tokenStream = createFieldTokenStream(tokenStreams,
+				fieldDescription);
+		
+		this.fieldDescription = fieldDescription;
+		String fieldName = fieldDescription.getName();
+		String delimiter = fieldDescription.getDelimiter();
+		Collection<Field> fields = new ArrayList<Field>();
+
+		Collection<FilterDescription> filterDescriptions = fieldDescription
+				.getFilterDescriptions();
+		tokenStream = getFilteredTokenStream(fieldName, tokenStream,
+				filterDescriptions);
+
+		Field.Store fieldStore = getFieldStore(fieldDescription.getStored());
+		Field.Index fieldIndex = getFieldIndex(fieldDescription.getIndex());
+		Field.TermVector fieldTermVector = getFieldTermVector(fieldDescription
+				.getTermVector());
+
+		if (fieldStore == Field.Store.YES || fieldStore == Field.Store.COMPRESS)
+			fields.addAll(createStoredFields(fieldName, tokenStream,
+					fieldStore, delimiter));
+
+		boolean omitTF = fieldDescription.getIndex().equals(FIELD_INDEX_NO_TF)
+				|| fieldDescription.getIndex().equals(FIELD_INDEX_NO_NORMS_TF);
+
+		if (fieldIndex != Field.Index.NO)
+			fields.add(createIndexedField(fieldName, tokenStream, fieldIndex,
+					fieldTermVector, omitTF));
+
+		return fields;
+	}
+
+	private TokenStream createFieldTokenStream(
+			Collection<TokenStream> tokenStreams,
+			FieldDescription fieldDescription) throws FieldBuildingException 
+	{
+		TokenStream tokenStream = null;
+		if (fieldDescription.getMerge())
+			tokenStream = getTokenStreamMerger(tokenStreams);
+		else if (tokenStreams.size() > 1)
+			tokenStream = new TokenStreamConcatenator(tokenStreams);
+		else if (tokenStreams.size() == 1)
+			tokenStream = tokenStreams.iterator().next();
+		return tokenStream;
+	}
+
+	private TokenStream getFilteredTokenStream(String fieldName,
+											   TokenStream tokenStream,
+											   Collection<FilterDescription> filterDescriptions)
+											   throws FieldBuildingException {
+		try {
+			return filterBuilder.filter(tokenStream, filterDescriptions);
+		} catch (FilterBuildingException e) {
+			throw createException(e);
+		}
+	}
+
+	private FieldBuildingException createException(Exception e) {
+		String message = "Can't build field " +  fieldDescription.getName()+
+						 " at line " + fieldDescription.getLineNumber();
+		return new FieldBuildingException(message, e);
+	}
+
+	private TokenStream getTokenStreamMerger(
+			Collection<TokenStream> tokenStreams) throws FieldBuildingException {
+		try {
+			return new TokenStreamMerger(tokenStreams);
+		} catch (IOException e) {
+			throw createException(e);
+		}
+
+	}
+
+	private Field createIndexedField(String fieldName, TokenStream tokenStream,
+			Index fieldIndex, TermVector fieldTermVector, boolean omitTF) {
+
+		Field field = new Field(fieldName, tokenStream, fieldTermVector);
+		if (fieldIndex == Field.Index.NOT_ANALYZED_NO_NORMS)
+			field.setOmitNorms(true);
+		field.setOmitTf(omitTF);
+
+		return field;
+	}
+
+	protected Collection<Field> createStoredFields(String fieldName,
+			TokenStream tokenStream, Store fieldStore, String delimiter)
+			throws FieldBuildingException {
+		Collection<Field> fields = new ArrayList<Field>();
+		try {
+			if (delimiter != null) {
+				String value = tokenStreamStringConcatenator
+						.tokenStreamToStringWithDelimiter(tokenStream,
+								delimiter);
+				fields.add(new Field(fieldName, value, fieldStore,
+						Field.Index.NO));
+			} else {
+				Token nextToken = tokenStream.next(new Token());
+				while (nextToken != null) {
+					String value = new String(nextToken.termBuffer(), 0,
+							nextToken.termLength());
+					fields.add(new Field(fieldName, value, fieldStore,
+							Field.Index.NO));
+					nextToken = tokenStream.next(nextToken);
+				}
+			}
+			tokenStream.reset();
+		} catch (IOException e) {
+			throw createException(e);
+		}
+		return fields;
+	}
+
+	protected Field.Index getFieldIndex(String index) throws FieldBuildingException {
+		if (index.equals(FIELD_INDEX_NO))
+			return Field.Index.NO;
+		else if (index.equals(FIELD_INDEX_YES))
+			return Field.Index.NOT_ANALYZED;
+		else if (index.equals(FIELD_INDEX_NO_NORMS))
+			return Field.Index.NOT_ANALYZED_NO_NORMS;
+		else if (index.equals(FIELD_INDEX_NO_NORMS_TF))
+			return Field.Index.NOT_ANALYZED_NO_NORMS;
+		else if (index.equals(FIELD_INDEX_NO_TF))
+			return Field.Index.NOT_ANALYZED;
+
+		throw createException(new IllegalArgumentException("unknown index parameter: " + index)); 
+		
+	}
+
+	protected Field.TermVector getFieldTermVector(String termVector) throws FieldBuildingException {
+		if (termVector.equals(FIELD_TERM_VECTOR_NO))
+			return Field.TermVector.NO;
+		else if (termVector.equals(FIELD_TERM_VECTOR_YES))
+			return Field.TermVector.YES;
+		else if (termVector.equals(FIELD_TERM_VECTOR_WITH_OFFSETS))
+			return Field.TermVector.WITH_OFFSETS;
+		else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS))
+			return Field.TermVector.WITH_POSITIONS;
+		else if (termVector.equals(FIELD_TERM_VECTOR_WITH_POSITIONS_OFFSETS))
+			return Field.TermVector.WITH_POSITIONS_OFFSETS;
+
+		throw createException(new IllegalArgumentException("unknown termVector parameter: "
+				+ termVector));
+	}
+
+	protected Field.Store getFieldStore(String store) throws FieldBuildingException {
+		if (store.equals(FIELD_STORE_NO))
+			return Field.Store.NO;
+		else if (store.equals(FIELD_STORE_YES))
+			return Field.Store.YES;
+		else if (store.equals(FIELD_STORE_COMPRESS))
+			return Field.Store.COMPRESS;
+
+		throw createException(new IllegalArgumentException("unknown stored parameter: " + store));
+	}
 
-    throw new IllegalArgumentException("unknown stored parameter: " + store);
-  }
 }

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FieldBuildingException.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+public class FieldBuildingException extends Exception {
+
+  /**
+   * 
+   */
+  private static final long serialVersionUID = 1L;
+
+  public FieldBuildingException(String message) {
+    super(message);
+  }
+  
+  public FieldBuildingException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public FieldBuildingException(Throwable cause) {
+    super(cause);
+  }
+}

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuilder.java Fri Jul 24 08:58:52 2009
@@ -20,115 +20,141 @@
 package org.apache.uima.lucas.indexer;
 
 import java.io.IOException;
-import java.util.List;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
 import java.util.Map;
 
 import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.snowball.SnowballFilter;
-import org.apache.uima.lucas.indexer.analysis.AdditionTokenFilter;
-import org.apache.uima.lucas.indexer.analysis.HypernymTokenFilter;
-import org.apache.uima.lucas.indexer.analysis.PositionFilter;
-import org.apache.uima.lucas.indexer.analysis.ReplaceFilter;
-import org.apache.uima.lucas.indexer.analysis.SplitterFilter;
-import org.apache.uima.lucas.indexer.analysis.UniqueFilter;
-import org.apache.uima.lucas.indexer.analysis.UpperCaseTokenFilter;
+import org.apache.uima.lucas.indexer.analysis.TokenFilterFactory;
+import org.apache.uima.lucas.indexer.mapping.FilterDescription;
+import org.apache.uima.lucas.indexer.mapping.Locateable;
 
 public class FilterBuilder {
 
-  private String[] stopwords;
-
-  private Map<String, List<String>> hypernyms;
-
-  private Map<String, Map<String, String>> mappings;
-
   public static final Logger LOGGER = Logger.getLogger(FilterBuilder.class);
 
-  public static final String POSITION_FIRST = "first";
-
-  public static final String POSITION_LAST = "last";
-
-  public FilterBuilder(String[] stopwords, Map<String, List<String>> hypernyms,
-          Map<String, Map<String, String>> mappings) {
-    super();
-    this.stopwords = stopwords;
-    this.hypernyms = hypernyms;
-    this.mappings = mappings;
-  }
-
-  public TokenStream filter(TokenStream tokenStream, AnnotationDescription description)
-          throws IOException {
-
-    String mappingFile = description.getMappingFile();
-    if (mappingFile != null) {
-      Map<String, String> mapping = mappings.get(mappingFile);
-      if (mapping == null)
-        throw new IllegalStateException(mappingFile + " not found!");
-      tokenStream = new ReplaceFilter(tokenStream, mapping);
-    }
-
-    // add filters
-    if (description.getPosition() != null) {
-      String position = description.getPosition();
-      if (position.equals(AnnotationDescription.FIRST_POSITION))
-        tokenStream = new PositionFilter(tokenStream, PositionFilter.FIRST_POSITION);
-      else if (position.equals(AnnotationDescription.LAST_POSITION))
-        tokenStream = new PositionFilter(tokenStream, PositionFilter.LAST_POSITION);
-    }
-
-    String prefix = description.getPrefix();
-    if (prefix != null)
-      tokenStream = new AdditionTokenFilter(tokenStream, prefix, AdditionTokenFilter.PREFIX);
-
-    String postfix = description.getPostfix();
-    if (postfix != null)
-      tokenStream = new AdditionTokenFilter(tokenStream, postfix, AdditionTokenFilter.POSTFIX);
-
-    String splitString = description.getSplitString();
-    if (splitString != null) {
-      tokenStream = new SplitterFilter(tokenStream, splitString);
-    }
+  private Map<String, TokenFilterFactory> cachedFactories;
+  private Locateable currentLocateable;
+  
+  public FilterBuilder(Map<String, TokenFilterFactory> predefinedFactories) {
+    cachedFactories = predefinedFactories;
+  }
 
-    if (description.getLowercase()) {
-      tokenStream = new LowerCaseFilter(tokenStream);
-    }
+  public TokenStream filter(TokenStream tokenStream,
+          Collection<FilterDescription> filterDescriptions) throws FilterBuildingException {
 
-    if (description.getUppercase()) {
-      tokenStream = new UpperCaseTokenFilter(tokenStream);
+    TokenStream filteredTokenStream = tokenStream;
+    for (FilterDescription filterDescription : filterDescriptions) {
+      String filterName = filterDescription.getName();
+      String factoryClassName = filterDescription.getFactoryClassName();
+      String className = filterDescription.getClassName();
+      currentLocateable = filterDescription;
+      
+      // use cached factory to create filter 
+      if (filterName != null && factoryClassName == null) {
+        TokenFilterFactory tokenFilterFactory = cachedFactories.get(filterName);
+        if (tokenFilterFactory == null)
+          throw createException("No factory registered for " + filterName
+                  + ". Please provide a factory");
+        
+        filteredTokenStream = createTokenFilter(filteredTokenStream, filterDescription, tokenFilterFactory);
+      }
+      // use single argument constructor of a token filter
+      else if (className != null)
+      {
+        filteredTokenStream = createTokenFilterWithClassName(filteredTokenStream, className);
+      }
+      // use factory for creating the filter and cache it if needed
+      else if (factoryClassName != null) 
+      {
+        TokenFilterFactory tokenFilterFactory = createTokenFilterFactory(factoryClassName);
+        filteredTokenStream = createTokenFilter(filteredTokenStream, filterDescription, tokenFilterFactory);
+        if( filterDescription.isReuseFactory() ){
+          if( filterName == null )
+            throw createException("Provide a name for factory reuse ");
+          cachedFactories.put(filterName, tokenFilterFactory);
+        }
+      }
     }
 
-    if (description.getStopwordRemove()) {
-      tokenStream = new StopFilter(tokenStream, stopwords);
-    }
-    if (description.getAddHypernyms()) {
-      if (hypernyms.size() == 0)
-        LOGGER.warn("hypernym adding activatet, but no hypernyms found!");
+    return filteredTokenStream;
+  }
 
-      tokenStream = new HypernymTokenFilter(tokenStream, hypernyms);
-    }
-    String snowballFilter = description.getSnowballFilter();
-    if (snowballFilter != null) {
-      tokenStream = new SnowballFilter(tokenStream, snowballFilter);
+  private TokenFilter createTokenFilter(TokenStream filteredTokenStream,
+          FilterDescription filterDescription, TokenFilterFactory tokenFilterFactory)
+          throws FilterBuildingException {
+    try {
+      return tokenFilterFactory.createTokenFilter(filteredTokenStream,
+              filterDescription.getProperties());
+    } catch (IOException e) {
+      throw createException("Can't build filter with description", e);
     }
+  }
 
-    if (description.getUnique()) {
-      tokenStream = new UniqueFilter(tokenStream);
+  private TokenFilterFactory createTokenFilterFactory(String factoryClassName)
+          throws FilterBuildingException {
+    try {
+      Class<?> clazz = Class.forName(factoryClassName);
+      Constructor<?>[] constructors = clazz.getConstructors();
+      for (Constructor<?> constructor : constructors) {
+        Class<?>[] parameters = constructor.getParameterTypes();
+        if (parameters.length == 0)
+          return (TokenFilterFactory) constructor.newInstance();
+      }
+    } catch (ClassNotFoundException e) {
+      throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+    } catch (IllegalArgumentException e) {
+      throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+    } catch (InstantiationException e) {
+      throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+    } catch (IllegalAccessException e) {
+      throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
+    } catch (InvocationTargetException e) {
+      throw createException("Can't instantiate TokenFilterFactory " + factoryClassName);
     }
+    throw new FilterBuildingException("Class " + factoryClassName + " has no public no argument constructor!");
+  }
 
-    return tokenStream;
+  private TokenFilter createTokenFilterWithClassName(TokenStream tokenStream, String className)
+          throws FilterBuildingException {
+    try {
+      Class<?> clazz = Class.forName(className);
+      Constructor<?>[] constructors = clazz.getConstructors();
+      for (Constructor<?> constructor : constructors) {
+        Class<?>[] parameters = constructor.getParameterTypes();
+        if (parameters.length == 1 && parameters[0].equals(TokenStream.class))
+          return (TokenFilter) constructor.newInstance(tokenStream);
+      }
+    } catch (ClassNotFoundException e) {
+      throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+    } catch (IllegalArgumentException e) {
+      throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+    } catch (InstantiationException e) {
+      throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+    } catch (IllegalAccessException e) {
+      throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+    } catch (InvocationTargetException e) {
+      throw new FilterBuildingException("Can't instantiate TokenFilter " + className, e);
+    }    
+    
+    throw createException("Class " + className + " has no public single argument constructor!");
   }
 
-  public String[] getStopwords() {
-    return stopwords;
+  private FilterBuildingException createException(String message, Throwable cause){
+	 String extendedMessage = message + " at line " + currentLocateable.getLineNumber();
+	 return new FilterBuildingException(extendedMessage, cause);
   }
 
-  public Map<String, List<String>> getHypernyms() {
-    return hypernyms;
+private FilterBuildingException createException(String message){
+	  String extendedMessage = message + " at line " + currentLocateable.getLineNumber();
+	  return new FilterBuildingException(extendedMessage);
   }
 
-  public Map<String, Map<String, String>> getMappings() {
-    return mappings;
+Map<String, TokenFilterFactory> getCachedFactories() {
+    return cachedFactories;
   }
+
 }

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/FilterBuildingException.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+public class FilterBuildingException extends Exception {
+
+  /**
+   * 
+   */
+  private static final long serialVersionUID = 1L;
+
+  public FilterBuildingException(String message) {
+    super(message);
+  }
+  
+  public FilterBuildingException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public FilterBuildingException(Throwable cause) {
+    super(cause);
+  }
+
+}

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProvider.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+import org.apache.lucene.index.IndexWriter;
+
+public interface IndexWriterProvider {
+
+  public IndexWriter getIndexWriter();
+  
+}

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/IndexWriterProviderImpl.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.uima.resource.DataResource;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.SharedResourceObject;
+
+public class IndexWriterProviderImpl implements IndexWriterProvider, SharedResourceObject{
+
+  public static final String USE_COMPOUND_FILE_FORMAT_PROPERTY = "useCompoundFileFormat";
+  public static final String RAMBUFFER_SIZE_PROPERTY = "RAMBufferSize";
+  public static final String INDEX_PATH_PROPERTY = "indexPath";
+  public static final String MAX_FIELD_LENGTH_PROPERTY = "maxFieldLength";
+  private static final String UNIQUE_INDEX_PROPERTY = "uniqueIndex";
+  public IndexWriter indexWriter;
+
+  public IndexWriter getIndexWriter() {
+    return indexWriter;
+  }
+
+  public void load(DataResource dataResource) throws ResourceInitializationException {
+    Properties properties = loadProperties(dataResource);
+    
+    String indexPath = getIndexPath(properties);
+    if (getUniqueIndexOrDefault(properties))
+      indexPath = createUniqueIndexPath(indexPath);
+
+    MaxFieldLength maxFieldLength = getMaxFieldLengthOrDefault(properties);
+    createIndexWriter(indexPath, maxFieldLength);
+    
+    Double ramBufferSize = getRAMBufferSizeOrDefault(properties);    
+    Boolean useCompoundFileFormat = getUseCompoundFormatOrDefault(properties);
+    configureIndexWriter(ramBufferSize, useCompoundFileFormat);
+  }
+
+  private Properties loadProperties(DataResource dataResource)
+          throws ResourceInitializationException {
+  
+    Properties properties = new Properties();
+    try {
+      properties.load(dataResource.getInputStream());
+    } catch (IOException e) {
+        throw new ResourceInitializationException(e);
+    }
+    
+    return properties;
+  }
+
+  private Boolean getUseCompoundFormatOrDefault(Properties properties) {
+    String useCompoundFileFormatAsString = properties.getProperty(USE_COMPOUND_FILE_FORMAT_PROPERTY);
+    Boolean useCompoundFileFormat = true;
+    if( useCompoundFileFormatAsString != null )
+      useCompoundFileFormat = new Boolean(useCompoundFileFormatAsString);
+    
+    return useCompoundFileFormat;
+  }
+
+  private String getIndexPath(Properties properties){
+    return properties.getProperty(INDEX_PATH_PROPERTY);
+  }
+  
+  private String createUniqueIndexPath(String indexPath){
+    String uniqueIndexPath = indexPath + "-" + getHostName() + "-" + getPID();
+    return uniqueIndexPath;
+  }
+
+  private Boolean getUniqueIndexOrDefault(Properties properties) {
+    String uniqueIndexAsString = properties.getProperty(UNIQUE_INDEX_PROPERTY);
+    Boolean uniqueIndex = false;
+    if( uniqueIndexAsString != null )
+      uniqueIndex = new Boolean(uniqueIndexAsString);
+    
+    return uniqueIndex;
+  }
+
+  private Double getRAMBufferSizeOrDefault(Properties properties) {
+    String ramBufferSizeAsString = properties.getProperty(RAMBUFFER_SIZE_PROPERTY);
+    Double ramBufferSize = IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB;
+    if( ramBufferSizeAsString != null )
+      ramBufferSize = new Double(ramBufferSizeAsString);
+    return ramBufferSize;
+  }
+  
+  private MaxFieldLength getMaxFieldLengthOrDefault(Properties properties) {
+    String maxFieldLengthAsString = properties.getProperty(MAX_FIELD_LENGTH_PROPERTY);
+    if( maxFieldLengthAsString == null )
+      return IndexWriter.MaxFieldLength.UNLIMITED; 
+    else{
+      Integer maxFieldLength = new Integer(maxFieldLengthAsString);
+      return new MaxFieldLength(maxFieldLength);
+    }        
+  }
+
+  private void createIndexWriter(String indexPath, MaxFieldLength maxFieldLength) throws ResourceInitializationException {
+    try {
+      indexWriter = new IndexWriter(indexPath, new StandardAnalyzer(), true, maxFieldLength);
+    } catch (CorruptIndexException e) {
+      throw new ResourceInitializationException(e);
+    } catch (LockObtainFailedException e) {
+      throw new ResourceInitializationException(e);    
+    } catch (IOException e) {
+      throw new ResourceInitializationException(e);    }
+  }
+
+  private void configureIndexWriter(Double ramBufferSize, Boolean useCompoundFileFormat) {
+    indexWriter.setRAMBufferSizeMB(ramBufferSize);
+    indexWriter.setUseCompoundFile(true);
+  }
+  
+  private String getPID() {
+    String id = ManagementFactory.getRuntimeMXBean().getName();
+    return id.substring(0, id.indexOf("@"));
+  }
+
+  private String getHostName() {
+    InetAddress address;
+    String hostName;
+    try {
+      address = InetAddress.getLocalHost();
+      hostName = address.getHostName();
+    } catch (UnknownHostException e) {
+      throw new IllegalStateException(e);
+    }
+    return hostName;
+  }
+
+}

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/Tokenizer.java Fri Jul 24 08:58:52 2009
@@ -25,6 +25,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.uima.lucas.indexer.mapping.AnnotationDescription;
 import org.apache.uima.lucas.indexer.util.TokenStreamStringConcatenator;
 
 public class Tokenizer {
@@ -49,8 +50,8 @@
           throws IOException {
     String tokenizer = description.getTokenizer();
     if (tokenizer != null && !tokenizer.equals(TOKENIZER_CAS)) {
-      String tokenStreamAsString =
-              tokenStreamStringConcatenator.tokenStreamToStringWithDelimiter(tokenStream, " ");
+      String tokenStreamAsString = tokenStreamStringConcatenator.tokenStreamToStringWithDelimiter(
+              tokenStream, " ");
       StringReader stringReader = new StringReader(tokenStreamAsString);
       if (tokenizer.equals(TOKENIZER_WHITESPACE))
         tokenStream = new WhitespaceTokenizer(stringReader);

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilter.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class AdditionFilter extends TokenFilter {
+
+  public static final Integer PREFIX = 0;
+
+  public static final Integer POSTFIX = 1;
+
+  private Integer position;
+
+  private String addition;
+
+  public AdditionFilter(TokenStream input, String addition, Integer position) {
+    super(input);
+    this.position = position;
+    this.addition = addition;
+  }
+
+  @Override
+  public Token next(Token aToken) throws IOException {
+    Token token = input.next(aToken);
+    if (token == null)
+      return null;
+
+    String tokenText = new String(token.termBuffer(), 0, token.termLength());
+    if (position.equals(PREFIX))
+      tokenText = addition.concat(tokenText);
+    else
+      tokenText = tokenText.concat(addition);
+
+    token.setTermBuffer(tokenText.toCharArray(), 0, tokenText.length());
+
+    return token;
+  }
+
+  public Integer getPosition() {
+    return position;
+  }
+
+  public String getAddition() {
+    return addition;
+  }
+}

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AdditionFilterFactory.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class AdditionFilterFactory implements TokenFilterFactory {
+
+	public static final String POSTFIX_POSITION = "postfix";
+	public static final String PREFIX_POSITION = "prefix";
+
+	public TokenFilter createTokenFilter(TokenStream tokenStream,
+			Properties properties) throws IOException {
+
+	  String postfix = properties.getProperty(POSTFIX_POSITION);	  
+	  String addition = null;
+    Integer position = null;
+    
+    if( postfix != null ){
+      position = AdditionFilter.POSTFIX;
+      addition = postfix;
+    }
+    else {
+      String prefix = properties.getProperty(PREFIX_POSITION);
+      if( prefix != null ){
+        position = AdditionFilter.PREFIX;
+        addition = prefix;        
+      }
+      else
+        throw new IllegalArgumentException("no postfix or prefix attribute provided");
+    }
+    
+    return new AdditionFilter(tokenStream, addition, position);
+	}
+
+	public void preloadResources(Properties properties) throws IOException {
+		// TODO Auto-generated method stub
+
+	}
+
+}

Modified: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStream.java Fri Jul 24 08:58:52 2009
@@ -21,10 +21,8 @@
 
 import java.io.IOException;
 import java.text.Format;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -45,6 +43,8 @@
 
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 /**
  * 
@@ -74,13 +74,11 @@
   private Iterator<Annotation> annotationIterator; // iterates over annotations
 
   private Iterator<FeatureStructure> featureStructureIterator; // iterates over feature structures
-
-  // stored in feature arrays of an
-  // annotation
+                                                               // stored in feature arrays of an
+                                                               // annotation
 
   private Iterator<String> featureValueIterator; // iterates over the features of a feature
-
-  // structure
+                                                 // structure
 
   private Annotation currentAnnotation;
 
@@ -108,23 +106,9 @@
    *          the type of the annotation
    * @throws CASException
    */
-  public AnnotationTokenStream(JCas cas, String sofaName, String typeName) throws CASException {
-    super();
-    jCas = cas.getView(sofaName);
-    this.featureNames = Collections.EMPTY_LIST;
-    this.featureFormats = Collections.EMPTY_MAP;
-
-    try {
-      annotationType = jCas.getTypeSystem().getType(typeName);
-      logger.debug(typeName + ", found: " + (annotationType != null));
-      logger.debug("featureNames: " + featureNames);
-      initializeIterators();
-    } catch (Exception e) {
-      IllegalArgumentException exc =
-              new IllegalArgumentException(e.getMessage() + " at type " + typeName);
-      exc.initCause(e);
-      throw exc;
-    }
+  public AnnotationTokenStream(JCas cas, String sofaName, String typeName) throws InvalidTokenSourceException {
+    this(cas, sofaName, typeName, null, Collections.EMPTY_LIST, null, 
+         Collections.EMPTY_MAP);
   }
 
   /**
@@ -144,30 +128,9 @@
    */
 
   public AnnotationTokenStream(JCas cas, String sofaName, String typeName, String featureName,
-          Format featureFormat) throws CASException {
-    super();
-    jCas = cas.getView(sofaName);
-    this.featureNames = new ArrayList<String>();
-    if (featureFormat != null) {
-      featureFormats = new HashMap<String, Format>();
-      featureFormats.put(featureName, featureFormat);
-    } else
-      this.featureFormats = Collections.EMPTY_MAP;
-
-    featureNames.add(featureName);
-
-    try {
-      annotationType = jCas.getTypeSystem().getType(typeName);
-      logger.debug(typeName + ", found: " + (annotationType != null));
-      logger.debug("featureNames: " + featureNames);
-      initializeIterators();
-    } catch (Exception e) {
-      IllegalArgumentException exc =
-              new IllegalArgumentException(e.getMessage() + " at type " + typeName);
-      exc.initCause(e);
-      throw exc;
-    }
-
+          Format featureFormat) throws InvalidTokenSourceException {
+    this(cas, sofaName, typeName, null, Lists.newArrayList(featureName), null, 
+         Maps.immutableBiMap(featureName, featureFormat));
   }
 
   /**
@@ -192,28 +155,8 @@
    */
   public AnnotationTokenStream(JCas cas, String sofaName, String typeName,
           List<String> featureNames, String delimiter, Map<String, Format> featureFormats)
-          throws CASException {
-    super();
-    jCas = cas.getView(sofaName);
-    this.featureNames = featureNames;
-    this.delimiter = delimiter;
-
-    if (featureFormats == null)
-      this.featureFormats = Collections.EMPTY_MAP;
-    else
-      this.featureFormats = featureFormats;
-
-    try {
-      annotationType = jCas.getTypeSystem().getType(typeName);
-      logger.debug(typeName + ", found: " + (annotationType != null));
-      logger.debug("featureNames: " + featureNames);
-      initializeIterators();
-    } catch (Exception e) {
-      IllegalArgumentException exc =
-              new IllegalArgumentException(e.getMessage() + " at type " + typeName);
-      exc.initCause(e);
-      throw exc;
-    }
+          throws InvalidTokenSourceException {
+    this(cas, sofaName, typeName, null, featureNames, delimiter, featureFormats);
   }
 
   /**
@@ -233,26 +176,8 @@
    * @throws CASException
    */
   public AnnotationTokenStream(JCas cas, String sofaName, String typeName,
-          List<String> featureNames, Map<String, Format> featureFormats) throws CASException {
-    super();
-    jCas = cas.getView(sofaName);
-    this.featureNames = featureNames;
-    if (featureFormats == null)
-      this.featureFormats = Collections.EMPTY_MAP;
-    else
-      this.featureFormats = featureFormats;
-
-    try {
-      annotationType = jCas.getTypeSystem().getType(typeName);
-      logger.debug(typeName + ", found: " + (annotationType != null));
-      logger.debug("featureNames: " + featureNames);
-      initializeIterators();
-    } catch (Exception e) {
-      IllegalArgumentException exc =
-              new IllegalArgumentException(e.getMessage() + " at type " + typeName);
-      exc.initCause(e);
-      throw exc;
-    }
+          List<String> featureNames, Map<String, Format> featureFormats) throws InvalidTokenSourceException {
+    this(cas, sofaName, typeName, null, featureNames, null, featureFormats);
   }
 
   /**
@@ -284,28 +209,8 @@
    * @throws CASException
    */
   public AnnotationTokenStream(JCas cas, String sofaName, String typeName, String featurePath,
-          List<String> featureNames, Map<String, Format> featureFormats) throws CASException {
-    super();
-    jCas = cas.getView(sofaName);
-    this.featurePath = featurePath;
-    this.featureNames = featureNames;
-    if (featureFormats == null)
-      this.featureFormats = Collections.EMPTY_MAP;
-    else
-      this.featureFormats = featureFormats;
-
-    try {
-      annotationType = jCas.getTypeSystem().getType(typeName);
-      logger.debug(typeName + ", found: " + (annotationType != null));
-      logger.debug("featurePath: " + featurePath);
-      logger.debug("featureNames: " + featureNames);
-      initializeIterators();
-    } catch (Exception e) {
-      IllegalArgumentException exc =
-              new IllegalArgumentException(e.getMessage() + " at type " + typeName);
-      exc.initCause(e);
-      throw exc;
-    }
+          List<String> featureNames, Map<String, Format> featureFormats) throws InvalidTokenSourceException {
+    this(cas, sofaName, typeName, featurePath, featureNames, null, featureFormats);
   }
 
   /**
@@ -339,9 +244,9 @@
    */
   public AnnotationTokenStream(JCas cas, String sofaName, String typeName, String featurePath,
           List<String> featureNames, String delimiter, Map<String, Format> featureFormats)
-          throws CASException {
+          throws InvalidTokenSourceException {
     super();
-    jCas = cas.getView(sofaName);
+
     this.featurePath = featurePath;
     this.featureNames = featureNames;
     this.delimiter = delimiter;
@@ -349,20 +254,57 @@
       this.featureFormats = Collections.EMPTY_MAP;
     else
       this.featureFormats = featureFormats;
+    
+    getSofaCas(cas, sofaName);
+    getTypeForName(typeName);
+    validate(annotationType, featureNames, featurePath);
+    
+    initializeIterators();
 
+  }
+
+  private void getTypeForName(String typeName) throws InvalidTokenSourceException{
+    annotationType = jCas.getTypeSystem().getType(typeName);
+    if( annotationType == null )
+      throw new InvalidTokenSourceException("Type " + typeName + " not found!");
+  }
+
+  private void getSofaCas(JCas cas, String sofaName) throws InvalidTokenSourceException {
     try {
-      annotationType = jCas.getTypeSystem().getType(typeName);
-      logger.debug(typeName + ", found: " + (annotationType != null));
-      logger.debug("featurePath: " + featurePath);
-      logger.debug("featureNames: " + featureNames);
-      initializeIterators();
-    } catch (Exception e) {
-      IllegalArgumentException exc =
-              new IllegalArgumentException(e.getMessage() + " at type " + typeName);
-      exc.initCause(e);
-      throw exc;
+      jCas = cas.getView(sofaName);
+    } catch (CASException e) {
+      throw new InvalidTokenSourceException(e);
     }
+  }
 
+  void validate(Type type, Collection<String> featureNames, String featurePath) throws InvalidTokenSourceException{
+    Type typeToValidate = findTypeWithPath(type, featurePath);
+    
+    for( String featureName: featureNames ){
+      Feature feature = typeToValidate.getFeatureByBaseName(featureName);
+      if( feature == null )
+        throw new InvalidTokenSourceException("Type " + typeToValidate.getName() + " has no feature " + featureName + ". featurePath: " + featurePath);
+    }
+  }
+  
+  private Type findTypeWithPath(Type type, String featurePath) throws InvalidTokenSourceException{
+    if( featurePath == null )
+      return type;
+    
+    String[] featurePathElements = featurePath.split("\\.");
+    Type currentType = type;
+    
+    for( String featurePathElement: featurePathElements ){
+      Feature feature = currentType.getFeatureByBaseName(featurePathElement);
+      if (feature == null)
+        throw new InvalidTokenSourceException("Type " + currentType.getName() + " has no feature " + featurePathElement);
+      
+      currentType = feature.getRange();
+      if (currentType.isArray())
+        currentType = currentType.getComponentType();
+    }
+    
+    return currentType;
   }
 
   @Override
@@ -376,8 +318,8 @@
           featureStructureIterator = createFeatureStructureIterator(currentAnnotation, featurePath);
         }
 
-        featureValueIterator =
-                createFeatureValueIterator(featureStructureIterator.next(), featureNames);
+        featureValueIterator = createFeatureValueIterator(featureStructureIterator.next(),
+                featureNames);
       }
 
       token.setStartOffset(currentAnnotation.getBegin());
@@ -389,10 +331,9 @@
 
     } catch (Throwable e) {
 
-      IOException ioException =
-              new IOException(e + " at type " + annotationType.getName() + " features "
-                      + featureNames + " featurePath " + featurePath + " sofa "
-                      + jCas.getViewName(), e);
+      IOException ioException = new IOException(e + " at type " + annotationType.getName()
+              + " features " + featureNames + " featurePath " + featurePath + " sofa "
+              + jCas.getViewName(), e);
       logger.error(ioException);
       throw ioException;
     }
@@ -409,9 +350,8 @@
   }
 
   protected void initializeIterators() {
-    annotationIterator =
-            Iterators.filter(jCas.getAnnotationIndex(annotationType).iterator(),
-                    new NotNullPredicate<Annotation>());
+    annotationIterator = Iterators.filter(jCas.getAnnotationIndex(annotationType).iterator(),
+            new NotNullPredicate<Annotation>());
 
     if (!annotationIterator.hasNext()) {
       featureStructureIterator = Iterators.emptyIterator();

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilter.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class ConcatFilter extends TokenFilter{
+
+	private int startOffset;
+	private String concatString;
+	private Token lastToken;
+	
+	protected ConcatFilter(TokenStream input, String concatString) {
+		super(input);
+		this.concatString = concatString;
+		startOffset = -1;
+	}
+	
+	@Override
+	public Token next(Token reusableToken) throws IOException {
+		String tokenText = concatenatedInputTokenTerms(reusableToken);
+		reusableToken.setTermBuffer(tokenText.toCharArray(), 0, tokenText.length());
+		reusableToken.setStartOffset(startOffset);
+		reusableToken.setEndOffset(lastToken.endOffset());
+		return reusableToken;
+	}
+
+	private String concatenatedInputTokenTerms(Token reusableToken) throws IOException {
+		StringBuffer stringBuffer = new StringBuffer();
+		Token nextToken = input.next(reusableToken);
+		lastToken = null;
+		
+		if( startOffset == -1 )
+			startOffset = nextToken.startOffset();
+		
+		while( nextToken != null ){
+			lastToken = nextToken;
+			stringBuffer.append(nextToken.term());
+			nextToken = input.next(reusableToken);
+			if( nextToken != null )
+				stringBuffer.append(concatString);
+		}
+		
+		return stringBuffer.toString();
+	}
+
+	protected String getConcatString() {
+		return concatString;
+	}
+
+}

Added: incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/main/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactory.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+
+public class ConcatFilterFactory implements TokenFilterFactory {
+
+	public static final String CONCAT_STRING_PARAMETER = "concatString";
+
+	public TokenFilter createTokenFilter(TokenStream tokenStream,
+			Properties properties) throws IOException {
+		
+		String concatString = properties.getProperty(CONCAT_STRING_PARAMETER);
+		return new ConcatFilter(tokenStream, concatString);
+	}
+
+	public void preloadResources(Properties properties) throws IOException {
+		// TODO Auto-generated method stub
+
+	}
+
+}