You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by mb...@apache.org on 2008/06/23 14:31:42 UTC

svn commit: r670541 [3/4] - in /incubator/uima/sandbox/trunk/ConceptMapper: ./ desc/ desc/analysis_engine/ desc/analysis_engine/aggregate/ desc/analysis_engine/primitive/ desc/collection_processing_engines/ doc/ resources/ resources/dict/ src/ src/main...

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryResource_impl.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryResource_impl.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryResource_impl.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryResource_impl.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,996 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.conceptMapper.support.dictionaryResource;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Enumeration;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.TreeMap;
+import java.util.Vector;
+import java.util.regex.Pattern;
+
+import org.apache.uima.analysis_engine.annotator.AnnotatorContext;
+import org.apache.uima.conceptMapper.Logger;
+import org.apache.uima.conceptMapper.support.dictionaryResource.annotatorAdaptor.AnnotatorAdaptor;
+import org.apache.uima.conceptMapper.support.tokens.TokenFilter;
+import org.apache.uima.conceptMapper.support.tokens.TokenNormalizer;
+import org.apache.uima.resource.DataResource;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.SharedResourceObject;
+import org.xml.sax.Attributes;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.helpers.XMLReaderFactory;
+
+/*
+ * @version $Revision: 1.5 $
+ * 
+ * @author Michael Tanenblatt <mt...@us.ibm.com> @author (c) Copyright 2005 IBM @author All Rights
+ * Reserved
+ */
+/**
+ * Implementation of a UIMA DictionaryResource
+ */
+
+public class DictionaryResource_impl implements DictionaryResource, SharedResourceObject {
+  /** Dictionary file loader. Uses an XML parser. */
+  protected DictLoader dictLoader;
+
+  /**
+   * Hashtable of first words. Contains a DictEntries object keyed on word string for the first word
+   * of every entry in the specified dictionary.
+   */
+  protected Hashtable<String, DictEntriesByLength> dictImpl;
+
+  /** Initial size of <code>dict</code> */
+  protected static final int NumOfInitialDictEntries = 500000;
+
+  boolean loaded;
+
+  /** Patterns to for matcher to replace SGML &amp;lt; entities */
+  private static final Pattern ltPattern = Pattern.compile("&lt;");
+
+  /** Patterns to for matcher to replace SGML &amp;gt; entities */
+  private static final Pattern gtPattern = Pattern.compile("&gt;");
+
+  /** Patterns to for matcher to replace SGML &amp;apos; entities */
+  private static final Pattern aposPattern = Pattern.compile("&apos;");
+
+  /** Patterns to for matcher to replace SGML &amp;quot; entities */
+  private static final Pattern quotPattern = Pattern.compile("&quot;");
+
+  /** Patterns to for matcher to replace SGML &amp;amp; entities */
+  private static final Pattern ampPattern = Pattern.compile("&amp;");
+
+  /** Configuration parameter key/label for the order independent lookup indicator */
+  public static final String PARAM_ORDERINDEPENDENTLOOKUP = "OrderIndependentLookup";
+
+  private boolean sortElements = false;
+
+  /** Configuration parameter key/label to indicate whether dictionary should be printed upon load */
+  public static final String PARAM_DUMPDICT = "PrintDictionary";
+
+  private boolean dumpDict = false;
+
+  /** Configuration parameter key/label for the case matching string */
+  public static final String PARAM_CASE_MATCH = "caseMatch";
+
+  /** Configuration parameter key/label for the stemmer class spec. If left out, no stemmer is used */
+  public static final String PARAM_STEMMER_CLASS = "Stemmer";
+
+  /**
+   * Configuration parameter key/label for the stemmer dictionary, passed into the stemmer's
+   * initialization method
+   */
+  public static final String PARAM_STEMMER_DICT = "StemmerDictionary";
+
+  private static final String PARAM_LANGID = "LanguageID";
+
+  private static final String DEFAULT_LANGID = "en";
+
+  public int entryNum = 0;
+
+  /**
+   * @param NumOfInitialDictEntries
+   * 
+   */
+  public DictionaryResource_impl() {
+    super();
+    dictImpl = new Hashtable<String, DictEntriesByLength>();
+    loaded = false;
+  }
+
+  /**
+   * @param NumOfInitialDictEntries
+   * 
+   */
+  public DictionaryResource_impl(int initialDictEntries) {
+    super();
+    dictImpl = new Hashtable<String, DictEntriesByLength>(initialDictEntries);
+    loaded = false;
+  }
+
+  /**
+   * @return Returns the dictLoader.
+   */
+  public DictLoader getDictLoader() {
+    return dictLoader;
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see org.apache.uima.conceptMapper.conceptMapper.support.DictionaryResource#getEntry(java.lang.String)
+   */
+  public DictEntriesByLength getEntries(String key) {
+    return dictImpl.get(key);
+  }
+
+  /**
+   * Create a new dictionary entry.
+   * 
+   * @param key
+   *          the key to index on
+   * @param phrase
+   *          the phrase to be entered in the dictionary
+   * @param length
+   *          the number of words in the phrase (>=1)
+   * @param props
+   *          the properties object for the dictionary entry
+   */
+  public void putEntry(String key, String text, ArrayList<String> elements, String unsorted,
+          int length, Properties props) {
+    DictEntriesByLength entry = getEntries(key);
+
+    if (entry == null) {
+      entry = new DictEntriesByLength_impl();
+      // System.err.println ("DICT adding: " + key + ", " + text);
+      dictImpl.put(key, entry);
+    }
+    // Iterator<String> elemIter = elements.iterator ();
+    // while (elemIter.hasNext ())
+    // {
+    // System.err.println("ENTRY ELEMENT: " + elemIter.next ());
+    // }
+    entry.putEntry(length, text, elements, unsorted, props);
+    // System.err.println("ENTRY: '" + text + "', PROPS: ");
+    // Enumeration propKeys = props.keys();
+    // while (propKeys.hasMoreElements())
+    // {
+    // String propKey = (String) propKeys.nextElement();
+    // System.err.println ("\t" + propKey + ": " + props.getProperty(propKey));
+    // }
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see org.apache.uima.resource.SharedResourceObject#load(org.apache.uima.resource.DataResource)
+   */
+  public void load(DataResource data) throws ResourceInitializationException {
+    // InputStream dictStream = null;
+    try {
+      Logger logger = new Logger("Dictionary Resource Loader", data.getLogger());
+      dictLoader = new DictLoader(logger, this, data);
+
+      /*
+       * At least for now, while info from the AnnotatorContext is not available here, must delay
+       * loading until able to access this info
+       * 
+       * //open input stream to data dictStream = data.getInputStream ();
+       * 
+       * dictLoader = new DictLoader (data.getLogger (), this); dictLoader.setDictionary
+       * (dictStream, NumOfInitialDictEntries);
+       */
+    } catch (Exception e) {
+      throw new ResourceInitializationException(e);
+    }
+    /*
+     * finally { if (dictStream != null) { try { dictStream.close (); } catch (IOException e) { } } }
+     */
+  }
+
+  public void loadDictionaryContents(AnnotatorContext aContext, Logger logger,
+          String tokenAnnotationName, String tokenTypeFeatureName, String tokenClassFeatureName,
+          String tokenizerDescriptor) throws ResourceInitializationException {
+
+    InputStream dictStream = null;
+    try {
+      dictLoader.setLogger(logger);
+      Boolean sortElementsParam = new Boolean(false);
+
+      sortElementsParam = (Boolean) aContext.getConfigParameterValue(PARAM_ORDERINDEPENDENTLOOKUP);
+      if (sortElementsParam == null) {
+        sortElements = false;
+      } else {
+        sortElements = sortElementsParam.booleanValue();
+      }
+
+      Boolean dumpDictParam = (Boolean) aContext.getConfigParameterValue(PARAM_DUMPDICT);
+      if (dumpDictParam == null) {
+        dumpDict = false;
+      } else {
+        dumpDict = dumpDictParam.booleanValue();
+      }
+
+      TokenNormalizer tokenNormalizer = new TokenNormalizer(aContext, logger);
+
+      // open input stream to data
+      dictStream = dictLoader.getInputStream();
+
+      TokenFilter tokenFilter = new TokenFilter(tokenAnnotationName, tokenTypeFeatureName,
+              tokenClassFeatureName, logger);
+      tokenFilter.initConfig(aContext);
+
+      String langID = (String) aContext.getConfigParameterValue(PARAM_LANGID);
+      if ((langID == null) || (langID == ""))
+      {
+        langID = DEFAULT_LANGID;
+      }
+      // System.out.print ("Loading Dictionary: '" + dictLoader.dataResource.getUri().toString() +
+      // "'...");
+      // System.out.print ("Loading Dictionary...");
+      logger.logInfo("Loading Dictionary...");
+      dictLoader.setDictionary(dictStream, NumOfInitialDictEntries, tokenAnnotationName,
+              tokenTypeFeatureName, tokenClassFeatureName, tokenizerDescriptor, tokenFilter,
+              tokenNormalizer, langID);
+      logger.logInfo("...done");
+      // System.out.println ("done");
+      // System.err.println("NEW DICT:\n" + toString());
+      setLoaded(true);
+      if (dumpDict)
+      {
+        System.err.println("BEGIN DUMPING DICTIONARY:");
+        System.err.println(toString());
+        System.err.println("DONE DUMPING DICTIONARY! ");
+      }
+
+    } catch (Exception e) {
+      throw new ResourceInitializationException(e);
+    } finally {
+      if (dictStream != null) {
+        try {
+          dictStream.close();
+        } catch (IOException e) {
+        }
+      }
+    }
+
+  }
+
+  /**
+   * @return Returns the loaded.
+   */
+  public boolean isLoaded() {
+    return loaded;
+  }
+
+  /**
+   * @param loaded
+   *          The loaded to set.
+   */
+  public void setLoaded(boolean loaded) {
+    this.loaded = loaded;
+  }
+
+  private static class DictEntriesByLength_impl implements DictEntriesByLength {
+    private static final long serialVersionUID = -8150386021246495622L;
+
+    private class ReverseOrderIntegerComparator implements Comparator<Integer>, Serializable {
+
+      /**
+       * 
+       */
+      private static final long serialVersionUID = -805437355806223406L;
+
+      /*
+       * (non-Javadoc)
+       * 
+       * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
+       */
+      public int compare(Integer left, Integer right) {
+        // reverse the order of parameters, to reverse the sorting order
+        return (right.compareTo(left));
+      }
+
+    }
+
+    TreeMap<Integer, DictEntries> entries;
+
+    public DictEntriesByLength_impl() {
+      super();
+      entries = new TreeMap<Integer, DictEntries>(new ReverseOrderIntegerComparator());
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntriesByLength#getEntries(int)
+     */
+    public DictEntries getEntries(int length) {
+      return entries.get(new Integer(length));
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntriesByLength#putEntry(int,
+     *      java.lang.String, java.util.Properties)
+     */
+    public void putEntry(int length, String text, ArrayList<String> elements, String unsorted,
+            Properties props) {
+      DictEntries entry = getEntries(length);
+      if (entry == null) {
+        entry = new DictEntriesImpl();
+        entries.put(new Integer(length), entry);
+      }
+      entry.putEntry(text, elements, unsorted, props);
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntriesByLength#getLongest()
+     */
+    public Integer getLongest() {
+      return entries.firstKey();
+    }
+
+    public Integer getShortest() {
+      return entries.lastKey();
+    }
+
+    public String toString() {
+      StringBuffer result = new StringBuffer();
+
+      int i = getLongest().intValue();
+      int last = getShortest().intValue();
+
+      while (i >= last) {
+        DictEntriesImpl entries = (DictEntriesImpl) getEntries(i);
+        if (entries != null) {
+          result.append("<DictEntriesByLength length='" + i + "'>\n");
+          result.append(entries.toString());
+          result.append("</DictEntriesByLength>\n");
+        }
+        i--;
+      }
+      return result.toString();
+    }
+  }
+
+  /**
+   * Private class for storing first words in the dict hashtable.
+   */
+  public static class DictEntriesImpl extends ArrayList<DictEntry> implements DictEntries {
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Create a new dictionary entry.
+     */
+    public DictEntriesImpl() {
+    }
+
+    /**
+     * Add a new phrase to an existing dictionary entry.
+     * 
+     * @param text
+     *          the text to be entered in the dictionary
+     * @param props
+     *          the properties object for the phrase
+     */
+    public void putEntry(String text, Collection<String> elements, String unsorted, Properties props) {
+      add(new DictEntryImpl(text, elements, unsorted, props));
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntries#getEntry(java.lang.String)
+     */
+    public ArrayList<DictEntry> getEntries() {
+      return this;
+    }
+
+    public String toString() {
+      StringBuffer result = new StringBuffer("<DictEntries>");
+      for (int i = 0; i < size(); i++) {
+        result.append(((DictEntryImpl) get(i)).toString());
+      }
+      result.append("</DictEntries>");
+      return result.toString();
+    }
+
+  }
+
+  public static class DictEntryImpl implements DictEntry {
+    private static final long serialVersionUID = -7723934333674544157L;
+
+    String text;
+
+    Collection<String> elements;
+
+    String unsorted;
+
+    Properties properties;
+
+    /**
+     * @param text
+     * @param elements
+     * @param unsorted
+     * @param properties
+     */
+    public DictEntryImpl(String text, Collection<String> elements, String unsorted,
+            Properties properties) {
+      super();
+      this.properties = properties;
+      this.unsorted = unsorted;
+      this.text = text;
+      this.elements = elements;
+    }
+
+    /**
+     * @return Returns the text.
+     */
+    public String getText() {
+      return text;
+    }
+
+    /**
+     * @param text
+     *          The text to set.
+     */
+    public void setText(String text) {
+      this.text = text;
+    }
+
+    /**
+     * @param properties
+     *          The properties to set.
+     */
+    public void setProperties(Properties properties) {
+      this.properties = properties;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntry#getProperties()
+     */
+    public Properties getProperties() {
+      return properties;
+    }
+
+    public String toString() {
+      StringBuffer result = new StringBuffer("<DictEntry Text ='" + getText() + "'>");
+      Enumeration<?> e = getProperties().propertyNames();
+
+      while (e.hasMoreElements()) {
+        String propName = (String) e.nextElement();
+        result.append("<property name='" + propName + "'>");
+        String item = getProperties().getProperty(propName);
+        result.append(item);
+        result.append("</property>\n");
+      }
+      result.append("</DictEntry>\n");
+      return result.toString();
+
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntry#getUnsorted()
+     */
+    public String getUnsorted() {
+      return unsorted;
+    }
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryResource.DictEntry#setUnsorted(java.lang.String)
+     */
+    public void setUnsorted(String unsorted) {
+      this.unsorted = unsorted;
+    }
+
+    public Collection<String> getElements() {
+      return elements;
+    }
+
+    public void setElements(Collection<String> elements) {
+      this.elements = elements;
+    }
+
+  }
+
+  /**
+   * Convert character entities in a string to the corresponding character. The set of entities
+   * handled includes:
+   * <ul>
+   * <li>&amp;amp;
+   * <li>&amp;lt;
+   * <li>&amp;gt;
+   * <li>&amp;apos;
+   * <li>&amp;quot;
+   * </ul>
+   * 
+   * @param input
+   *          the string to process.
+   * 
+   * @return the string with converted entities
+   */
+  protected String convertEntities(String input) {
+    String result = ltPattern.matcher(input).replaceAll("<");
+    result = gtPattern.matcher(result).replaceAll(">");
+    result = aposPattern.matcher(result).replaceAll("'");
+    result = quotPattern.matcher(result).replaceAll("\"");
+    result = ampPattern.matcher(result).replaceAll("&");
+
+    return (result);
+  }
+
+  /**
+   * Private class to load the dictionary file. Extends the org.xml.sax.helpers.DefaultHandler for
+   * XML parsing.
+   */
+  private class DictLoader extends DefaultHandler {
+    /** Default parser name. */
+    protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser";
+
+    /** Default name of element that contains dictionary records. */
+    protected static final String DEFAULT_TOKEN_ELEM = "token";
+
+    /** Default name of element that contains variant form */
+    protected static final String DEFAULT_VARIANT_ELEM = "variant";
+
+    /**
+     * Default name of attribute in the token element that contains the key for the entry (i.e., the
+     * surface or variant form that will be found in the document).
+     */
+    protected static final String DEFAULT_KEY_ATTR = "base";
+
+    /** Default name of element that contains part of speech */
+    // protected static final String DEFAULT_POS_ATTR = "POS";
+    /** The XML parser that parses the dictionary file. */
+    private XMLReader parser = null;
+
+    /** The name of the element that contains the dictionary records. */
+    private String token_elem = DEFAULT_TOKEN_ELEM;
+
+    /**
+     * The name of the element that contains a variant form for the current entry
+     */
+    private String variant_elem = DEFAULT_VARIANT_ELEM;
+
+    /** The name of the attribute that contains the key. */
+    private String key_attribute = DEFAULT_KEY_ATTR;
+
+    /** The name of the attribute that contains the part of speech. */
+    // private String pos_attribute = DEFAULT_POS_ATTR;
+    /** Count of number of dictionary entries loaded. */
+    private int term_cnt = 0;
+
+    /** Reference to UIMA logger. */
+    private Logger logger;
+
+    /** The hashtable built while parsing the dictionary file. */
+    private DictionaryResource dict;
+
+    /** Properties for current canonical form */
+    private Properties props;
+
+    private AnnotatorAdaptor adaptor;
+
+    private String tokenAnnotationName;
+
+    private String tokenizerDescriptor;
+
+    private TokenFilter tokenFilter;
+
+    private TokenNormalizer tokenNormalizer;
+
+    /**
+     * needed to access input stream, since cannot load external dict resource until TAE config
+     * params are available to set up tokenizer correctly
+     */
+    DataResource dataResource;
+
+    private Vector<DictionaryToken> result;
+
+    /**
+     * Create a dictionary loader.
+     * 
+     * @param The
+     *          UIMA logger.
+     * 
+     * @throws Exception
+     *           if XML parser cannot be created.
+     */
+    public DictLoader(Logger log, DictionaryResource dict, DataResource data) throws Exception {
+      this.setLogger(log);
+      this.dict = dict;
+      this.dataResource = data;
+
+      // create parser
+      try {
+        parser = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME);
+      } catch (Exception e) {
+        log.logError("Unable to instantiate dictionary parser (" + DEFAULT_PARSER_NAME + ")");
+        throw (e);
+      }
+      parser.setContentHandler(this);
+      parser.setErrorHandler(this);
+
+    }
+
+    protected String getTokenAnnotationName() {
+      return tokenAnnotationName;
+    }
+
+    protected void setTokenAnnotationName(String tokenAnnotationName) {
+      this.tokenAnnotationName = tokenAnnotationName;
+    }
+
+    protected String getTokenizerDescriptor() {
+      return tokenizerDescriptor;
+    }
+
+    protected void setTokenizerDescriptor(String tokenizerDescriptor) {
+      this.tokenizerDescriptor = tokenizerDescriptor;
+    }
+
+    protected void setTokenFilter(TokenFilter tokenFilter) {
+      this.tokenFilter = tokenFilter;
+    }
+
+    protected TokenFilter getTokenFilter() {
+      return tokenFilter;
+    }
+
+    protected void setTokenNormalizer(TokenNormalizer tokenNormalizer) {
+      this.tokenNormalizer = tokenNormalizer;
+    }
+
+    protected TokenNormalizer getTokenNormalizer() {
+      return tokenNormalizer;
+    }
+
+    /**
+     * @return Returns the log.
+     */
+    public Logger getLogger() {
+      return logger;
+    }
+
+    /**
+     * @param log
+     *          The log to set.
+     */
+    public void setLogger(Logger logger) {
+      this.logger = logger;
+    }
+
+    /**
+     * Start element. This method does most of the work of building the hashtable.
+     * 
+     */
+    public void startElement(String uri, String local, String raw, Attributes attrs)
+            throws SAXException {
+
+      String key = null;
+      DictionaryToken token = null;
+      int length = 0;
+
+      if (raw.equals(token_elem)) { // starting new token entry
+        if (attrs != null) {
+
+          props = new Properties();
+          int attrCount = attrs.getLength();
+          for (int i = 0; i < attrCount; i++) {
+            props.setProperty(attrs.getQName(i), convertEntities(attrs.getValue(i)));
+          }
+        }
+      } else if (raw.equals(variant_elem)) { // variant for current token
+        if (attrs != null) {
+          int attrCount = attrs.getLength();
+          ArrayList<String> tokens = new ArrayList<String>();
+
+          // if this variant contains its own POS info, save token level POS info and set props to
+          // contain variant's
+          Properties variantProperties = new Properties(props);
+          // logger.logInfo("" + entryNum++);
+          //
+          // System.err.println("" + entryNum++);
+          for (int i = 0; i < attrCount; i++) {
+            if (attrs.getQName(i).equals(key_attribute)) { // key attribute?
+
+              adaptor.runCPM(convertEntities(attrs.getValue(i)));
+
+              //if (dumpDict)
+              //{
+              //  System.err.println ("Adaptor done, result size: " + result.size() + "\nresult:\n");
+              //  for (DictionaryToken resultItem : result)
+              //  {
+              //    System.err.println ("  " + resultItem.getText()); 
+              //  }
+              //}
+              
+              Iterator<DictionaryToken> tokenIter = result.iterator();
+              token = null;
+
+              while (tokenIter.hasNext()) {
+                token = (DictionaryToken) tokenIter.next();
+                //if (dumpDict)
+                //{
+                //  System.err.println ("TOKEN CLASS: '" + token.getTokenClass() + "', TOKEN TYPE: '" + token.getType() + "'");
+                //}
+                if (tokenFilter.isOK_Token(token, tokenNormalizer)) {
+                  break;
+                }
+              }
+
+              if (token == null) {
+                return;
+              }
+
+              key = new String(tokenNormalizer.normalize(token.getText()));
+              //if (dumpDict)
+              //{
+              //  System.err.println ("variant token key:" + key);
+              //}
+
+              tokens.add(key);
+              length = 1;
+              while (tokenIter.hasNext()) {
+                token = (DictionaryToken) tokenIter.next();
+                String tokenText = tokenNormalizer.normalize(token.getText());
+
+                if (tokenFilter.isOK_Token(token, tokenNormalizer)) {
+                  key += " " + tokenText;
+                  tokens.add(tokenText);
+                  length++;
+                  
+                  //if (dumpDict)
+                  //{
+                  //  System.err.println (" variant token aux:" + tokenText);
+                  //}
+                }
+                //else
+                //{
+                //  if (dumpDict)
+                //  {
+                //    System.err.println (" SKIPPING: variant token aux:" + tokenText);
+                //  }
+                //}
+              }
+            } else {
+              variantProperties.setProperty(attrs.getQName(i), convertEntities(attrs.getValue(i)));
+            }
+
+          }
+
+          String[] elements = (String[]) tokens.toArray(new String[tokens.size()]);
+
+          String unsorted = null;
+
+          if (sortElements) {
+            unsorted = stringTogetherTokens(elements);
+            Arrays.sort(elements);
+          }
+
+          String tokenString = stringTogetherTokens(elements);
+          //if (dumpDict)
+          //{
+          //  System.err.println ("token string: " + tokenString);
+          //}
+
+
+          // add to dictionary
+          if (sortElements) {
+            for (int i = 0; i < tokens.size(); i++) {
+              dict.putEntry((String) tokens.get(i), tokenString, tokens, unsorted, tokens.size(),
+                      variantProperties);
+              // System.err.println ("adding props for:" + tokenString);
+              // Enumeration propKeys = variantProperties.keys();
+              // while (propKeys.hasMoreElements())
+              // {
+              // String propKey = (String) propKeys.nextElement();
+              // System.err.println ("\t" + propKey + ": " +
+              // variantProperties.getProperty(propKey));
+              // }
+
+            }
+          } else {
+            dict.putEntry((String) tokens.get(0), tokenString, tokens, unsorted, tokens.size(),
+                    variantProperties);
+          }
+          term_cnt++;
+          if ((term_cnt % 10000) == 0) {
+            getLogger().logInfo("processed " + term_cnt + " entries");
+            System.err.println("processed " + term_cnt + " entries");
+          }
+        }
+      }
+    }
+
+    /**
+     * @param elements
+     * @return
+     */
+    private String stringTogetherTokens(String[] elements) {
+      StringBuffer tokenString = new StringBuffer();
+
+      for (int i = 0; i < elements.length; i++) {
+        if (i > 0) {
+          tokenString.append(" ");
+        }
+        tokenString.append(elements[i]);
+      }
+      return tokenString.toString();
+    }
+
+    //
+    // ErrorHandler methods
+    //
+    /** Warning. */
+    public void warning(SAXParseException ex) throws SAXException {
+      getLogger().logWarning(errorString("Warning", ex));
+    } // warning(SAXParseException)
+
+    /**
+     * Error. public void error (SAXParseException ex) throws SAXException { getLogger ().log
+     * (Level.SEVERE, errorString ("Error", ex)); } // error(SAXParseException)
+     */
+    /** Fatal error. */
+    public void fatalError(SAXParseException ex) throws SAXException {
+      getLogger().logError(errorString("Fatal Error", ex));
+      throw ex;
+    } // fatalError(SAXParseException)
+
+    /** Prints the error message. */
+    protected String errorString(String type, SAXParseException ex) {
+      String errorMsg = "[" + type + "]";
+      if (ex == null) {
+        return errorMsg + "!!!";
+      }
+      String systemId = ex.getSystemId();
+      if (systemId != null) {
+        int index = systemId.lastIndexOf('/');
+        if (index != -1)
+          systemId = systemId.substring(index + 1);
+        errorMsg += systemId;
+      }
+      errorMsg += ":" + ex.getLineNumber() + ":" + ex.getColumnNumber() + ": " + ex.getMessage();
+      return errorMsg;
+    }
+
+    /**
+     * Use the tokenizer specified in 'tokenizerDescriptor' to load the specified dicitonary file.
+     * The dictonary file must be in the format specified above. A new <code>dict</code> hashtable
+     * is created with a <code>DictEntries</code> object for each unique first word in the base
+     * fields in the dictionary file. The <code>dict</code> hashtable is keyed off of the first
+     * word. The <code>DictEntries</code> for a first word contains a hashtable of
+     * <code>DictEntry</code> objects for every phrase in the base fields of the dictionary file
+     * started by the first word. The phrase hashtable is keyed off of the entire phrase.
+     * 
+     * @param tokenizerDescriptor
+     * @param tokenAnnotationName
+     * @param tokenFilter
+     * 
+     * @param dictFile
+     *          the fully specified filename of the dictionary file to load.
+     * 
+     * @param NumOfInitialDictEntries
+     *          initial size of hashtable to create
+     * 
+     * @exception java.io.IOException
+     *              if dictionary file cannot be loaded or some other initialization error occurs.
+     */
+    public void setDictionary(InputStream dictStream, int initialDictEntries,
+            String tokenAnnotationName, String tokenTypeFeatureName, String tokenClassFeatureName,
+            String tokenizerDescriptor, TokenFilter tokenFilter, TokenNormalizer tokenNormalizer, String langID)
+            throws DictionaryLoaderException {
+      term_cnt = 0;
+      setTokenAnnotationName(tokenAnnotationName);
+      setTokenizerDescriptor(tokenizerDescriptor);
+      setTokenFilter(tokenFilter);
+      setTokenNormalizer(tokenNormalizer);
+      result = new Vector<DictionaryToken>();
+
+      getLogger().logInfo("Loading dictionary");
+      try {
+        adaptor = new AnnotatorAdaptor(getTokenizerDescriptor(), result, tokenAnnotationName,
+                tokenFilter, langID);
+        adaptor.initCPM();
+
+        parser.parse(new InputSource(dictStream));
+      } catch (SAXException e) {
+        getLogger().logError("Parse error occurred - " + e.getMessage());
+        throw new DictionaryLoaderException();
+      } catch (IOException e) {
+        throw new DictionaryLoaderException();
+      }
+      getLogger().logInfo("Finished loading " + term_cnt + " entries");
+    }
+
+    public InputStream getInputStream() throws IOException {
+      return dataResource.getInputStream();
+    }
+
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see org.apache.uima.conceptMapper.support.DictionaryResource#NewDictionaryResource(int)
+   */
+  public DictionaryResource NewDictionaryResource(int initialDictEntries) {
+    return new DictionaryResource_impl(initialDictEntries);
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see org.apache.uima.conceptMapper.support.DictionaryResource#keys()
+   */
+  public Enumeration<String> keys() {
+    return dictImpl.keys();
+  }
+
+  public String toString() {
+    StringBuffer result = new StringBuffer();
+
+    Enumeration<String> e = keys();
+
+    while (e.hasMoreElements()) {
+      String key = e.nextElement();
+      result.append("<DictionaryItem key='" + key + "'>\n");
+      DictEntriesByLength_impl item = (DictEntriesByLength_impl) getEntries(key);
+      result.append(item.toString());
+      result.append("</DictionaryItem>\n");
+    }
+    return result.toString();
+  }
+
+  public void serializeEntries(FileOutputStream output) throws IOException {
+    ObjectOutputStream oos = new ObjectOutputStream(output);
+    oos.writeObject(this.dictImpl);
+    oos.close();
+  }
+
+}
\ No newline at end of file

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryToken.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryToken.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryToken.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/DictionaryToken.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.conceptMapper.support.dictionaryResource;
+
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.text.AnnotationFS;
+
+public class DictionaryToken {
+
+  private String text;
+  private Integer type;
+  private String tokenClass;
+
+  private boolean tokenTypeFeatureDefined;
+  private boolean tokenClassFeatureDefined;
+  
+  
+  public DictionaryToken(AnnotationFS annotation, Feature tokenTypeFeature, Feature tokenClassFeature) {
+    super();
+    if (tokenTypeFeature == null)
+    {
+      this.setTokenTypeFeatureDefined(false);
+    }
+    else
+    {
+      this.setTokenTypeFeatureDefined(true);
+      this.setType(annotation.getIntValue(tokenTypeFeature));
+    }
+    if (tokenClassFeature == null)
+    {
+      this.setTokenClassFeatureDefined(false);
+    }
+    else
+    {
+      this.setTokenClassFeatureDefined(true);
+      this.setTokenClass(annotation.getStringValue(tokenClassFeature));
+    }
+    this.setText(annotation.getCoveredText());
+  }
+
+  public String getText() {
+    return text;
+  }
+
+  public void setText(String text) {
+    this.text = text;
+  }
+
+  public String getTokenClass() {
+    return tokenClass;
+  }
+
+  public void setTokenClass(String tokenClass) {
+    this.tokenClass = tokenClass;
+  }
+
+  public Integer getType() {
+    return type;
+  }
+
+  public void setType(Integer type) {
+    this.type = type;
+  }
+
+  public boolean isTokenTypeFeatureDefined() {
+    return tokenTypeFeatureDefined;
+  }
+
+  public void setTokenTypeFeatureDefined(boolean tokenTypeFeatureDefined) {
+    this.tokenTypeFeatureDefined = tokenTypeFeatureDefined;
+  }
+
+  public boolean isTokenClassFeatureDefined() {
+    return tokenClassFeatureDefined;
+  }
+
+  public void setTokenClassFeatureDefined(boolean tokenClassFeatureDefined) {
+    this.tokenClassFeatureDefined = tokenClassFeatureDefined;
+  }
+
+}

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/annotatorAdaptor/AnnotatorAdaptor.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/annotatorAdaptor/AnnotatorAdaptor.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/annotatorAdaptor/AnnotatorAdaptor.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/dictionaryResource/annotatorAdaptor/AnnotatorAdaptor.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.conceptMapper.support.dictionaryResource.annotatorAdaptor;
+
+import java.io.IOException;
+import java.util.Vector;
+
+import org.apache.uima.UIMAFramework;
+import org.apache.uima.analysis_engine.AnalysisEngine;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryLoaderException;
+import org.apache.uima.conceptMapper.support.dictionaryResource.DictionaryToken;
+import org.apache.uima.conceptMapper.support.tokens.TokenFilter;
+import org.apache.uima.conceptMapper.support.tokens.UnknownTypeException;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceSpecifier;
+import org.apache.uima.util.InvalidXMLException;
+import org.apache.uima.util.XMLInputSource;
+
+public class AnnotatorAdaptor {
+  private ResourceSpecifier aeSpecifier;
+
+  private AnalysisEngine ae;
+
+  private CAS cas;
+
+  private Vector<DictionaryToken> result;
+
+  private TokenFilter tokenFilter;
+
+  private String tokenTypeName;
+
+  private Type tokenType;
+  private Feature tokenTypeFeature;
+  private Feature tokenClassFeature;
+
+  // private static Set<Type> tokenAnnotationSubTypes = null;
+  private boolean typeSystemInitialized = false;
+
+  private String langID;
+
+  public AnnotatorAdaptor(String analysisEngineDescriptorPath, Vector<DictionaryToken> result,
+          String tokenTypeName, TokenFilter tokenFilter, String langID)
+          throws DictionaryLoaderException {
+    super();
+    try {
+      aeSpecifier = UIMAFramework.getXMLParser().parseResourceSpecifier(
+              new XMLInputSource(analysisEngineDescriptorPath));
+      this.tokenTypeName = tokenTypeName;
+      this.tokenTypeFeature = tokenFilter.getTokenTypeFeature();
+      this.tokenClassFeature = tokenFilter.getTokenClassFeature();
+      this.tokenFilter = tokenFilter;
+      this.langID = langID;
+      this.result = result;
+    } catch (InvalidXMLException e) {
+      throw new DictionaryLoaderException(e);
+    } catch (IOException e) {
+      throw new DictionaryLoaderException(e);
+    }
+  }
+
+  public void initCPM() throws DictionaryLoaderException {
+    try {
+      ae = UIMAFramework.produceAnalysisEngine(aeSpecifier);
+      cas = ae.newCAS();
+    } catch (ResourceInitializationException e) {
+      throw new DictionaryLoaderException(e);
+    }
+  }
+
+  public void runCPM(String text) {
+    cas.setDocumentText(text);
+    cas.setDocumentLanguage(langID);
+    try {
+      ae.process(cas);
+    } catch (AnalysisEngineProcessException e) {
+      e.printStackTrace();
+    }
+    processCAS(cas);
+    cas.reset();
+  }
+
+  private void processCAS(CAS cas) {
+    if (cas == null) {
+    } else {
+      try {
+        // System.err.println ("processCAS(), doc text: " + cas.getDocumentText());
+        result.clear();
+        if (!typeSystemInitialized) {
+          TypeSystem typeSystem = cas.getTypeSystem();
+          tokenFilter.initTypes(typeSystem, false);
+          tokenType = typeSystem.getType(tokenTypeName);
+          typeSystemInitialized = true;
+        }
+
+        FSIterator tokenIter = cas.getIndexRepository().getAllIndexedFS(tokenType);
+        while (tokenIter.hasNext()) {
+          AnnotationFS annotation = (AnnotationFS) tokenIter.next();
+          result.add(new DictionaryToken(annotation, tokenTypeFeature, tokenClassFeature));
+        }
+      } catch (UnknownTypeException e) {
+        System.err.println(e.getMessage());
+        e.printStackTrace();
+      }
+    }
+  }
+}

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/JunkStemmer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/JunkStemmer.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/JunkStemmer.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/JunkStemmer.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.conceptMapper.support.stemmer;
+
+/*
+ * Implements the Stemmer interface, but does nothing
+ */
+public class JunkStemmer implements Stemmer {
+
+  /**
+   * 
+   */
+  public JunkStemmer() {
+    super();
+  }
+
+  public String stem(String token) {
+    System.err.println("Junk Stemming: " + token);
+    return (token);
+  }
+
+  public void initialize(String dictionary) {
+    // do nothing
+  }
+}

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/Stemmer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/Stemmer.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/Stemmer.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/stemmer/Stemmer.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.conceptMapper.support.stemmer;
+
+import java.io.FileNotFoundException;
+import java.text.ParseException;
+
+/**
+ * Interface for stemmer. Two methods are defined:
+ * 
+ * {@link #stem stem()}, which takes a single token in as a <code>String</code> and returns the
+ * stemmed token as a <code>String</code>. {@link #stem initialize ()}, which takes a dictionary
+ * path as a <code>String</code> loads it.
+ * 
+ * 
+ */
+public interface Stemmer {
+
+  /**
+   * Take a single token in as a <code>String</code>, stem the token, and return the stemmed
+   * token as a <code>String</code>.
+   * 
+   * @param token
+   *          the input token to stem
+   * @return the stemmed token
+   */
+  public String stem(String token);
+
+  /**
+   * Initialize the stemmer with a dictionary
+   */
+  public void initialize(String dictionary) throws FileNotFoundException, ParseException;
+
+}

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/OffsetTokenizer.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/OffsetTokenizer.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/OffsetTokenizer.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/OffsetTokenizer.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,480 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.conceptMapper.support.tokenizer;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.uima.analysis_engine.ResultSpecification;
+import org.apache.uima.analysis_engine.annotator.AnnotatorConfigurationException;
+import org.apache.uima.analysis_engine.annotator.AnnotatorContext;
+import org.apache.uima.analysis_engine.annotator.AnnotatorInitializationException;
+import org.apache.uima.analysis_engine.annotator.AnnotatorProcessException;
+import org.apache.uima.analysis_engine.annotator.JTextAnnotator_ImplBase;
+import org.apache.uima.conceptMapper.support.stemmer.Stemmer;
+import org.apache.uima.jcas.JCas;
+
+/**
+ * Simple class to tokenize a string (similar to <code>java.util.StringTokenizer</code>), except
+ * that this tokenizer returns
+ * {@link org.apache.uima.conceptMapper.support.TokenAnnotation TokenAnnotation} objects, which, in
+ * addition to the token text string, also contain the start and end offsets of the token in the
+ * original string.
+ * <p>
+ * 
+ * The tokenizer will optionally perform stemming and case normalization on the tokens, and the set
+ * of characters that delimit tokens may be specified. The default stemmer is the Snowball Porter
+ * stemmer, but any stemmer may be supplied to the tokenizer as long as it implements the
+ * {@link org.apache.uima.conceptMapper.support.Stemmer Stemmer}interface.
+ * 
+ */
+public class OffsetTokenizer  extends JTextAnnotator_ImplBase {
+
+  /** The input text string being tokenized */
+  private String text;
+
+  /** The stemmer that will perform the stemming. */
+  private Stemmer stemmer = null;
+
+  //private Logger logger;
+
+  /** Configuration parameter key/label for the case matching string */
+  public static final String PARAM_CASE_MATCH = "caseMatch";
+
+  /** Configuration parameter key/label for the stemmer class spec */
+  public static final String PARAM_STEMMER_CLASS = "Stemmer";
+
+  /** Configuration parameter key/label for the token delimiters string */
+  public static final String PARAM_TOKEN_DELIM = "tokenDelimiters";
+
+  /** The current offset into {@link TextToken.text} */
+  private int offset;
+
+  /** The total length of {@link TextToken.text} */
+  private int length;
+
+  /** Base whitespace token delimiters */
+  private String wsDelim = " \t\n\r\f";
+
+  /**
+   * The extra specifiable delimiters used by the tokenizer to separate the input string into
+   * individual tokens.
+   */
+  private String extraDelim = ",-/();:";
+
+  /**
+   * The delimiters used by the tokenizer to separate the input string into individual tokens.
+   */
+  private String delim = wsDelim + extraDelim;
+
+  /** Case folding flag for folding tokens with initial cap. */
+  private boolean caseFoldInitCap = false;
+
+  /** Case folding flag for folding tokens with at least one digit character. */
+  private boolean caseFoldDigit = false;
+
+  /**
+   * Case folding flag for folding all tokens (subsumes {@link #caseFoldInitCap caseFoldInitCap}and
+   * {@link #caseFoldDigit caseFoldDigit}).
+   */
+  private boolean caseFoldAll = false;
+
+  /** Stemming on/off flag. If true, tokens will be stemmed by the tokenizer. */
+  private boolean stemTokens = false;
+
+  /**
+   * regular expression pattern used to identify terms with an initial capital letter followed by
+   * all lowercase letters.
+   */
+  private Pattern capPat = null;
+
+  /**
+   * regular expression pattern used to identify terms with at least one digit character in them.
+   */
+  private Pattern hasDigit = null;
+
+  /**
+   * Create a new <code>OffsetTokenizer</code>. Initializes the default stemmer and sets up the
+   * regular expressions for the various case folding options.
+   */
+  public OffsetTokenizer() {
+    try {
+      capPat = Pattern.compile("^[A-Z][a-z]+$");
+      hasDigit = Pattern.compile("[0-9]");
+    } catch (PatternSyntaxException e) {
+      e.printStackTrace();
+    }
+
+  }
+
+
+  /**
+   * @return Returns the text.
+   */
+  public String getText() {
+    return text;
+  }
+
+  /**
+   * Set the text to tokenize. After this method is called, the next call to
+   * {@link #nextToken nextToken} will return the first token from the input string
+   * {@link TextToken.text}
+   */
+  public void setText(String text) {
+    this.text = text;
+    offset = 0;
+    length = getText().length();
+  }
+  
+  
+  /**
+   * @return Returns the stemmer.
+   */
+  public Stemmer getStemmer() {
+    return stemmer;
+  }
+
+  /**
+   * @param stemmer
+   *          The stemmer to set.
+   */
+  public void setStemmer(Stemmer stemmer) {
+    this.stemmer = stemmer;
+  }
+
+  public TokenAnnotation newToken(JCas jcas) {
+    return new TokenAnnotation(jcas);
+  }
+
+
+
+  public TokenAnnotation nextToken(JCas jcas) {
+    StringBuffer token = new StringBuffer();
+    // System.out.println("in NextToken");
+    // skip any delimiters
+    while ((offset < length) && (getDelim().indexOf(getText().charAt((offset))) >= 0)) {
+      offset += 1;
+    }
+
+    if (offset < length) {
+      int start = offset;
+      while ((offset < length) && (getDelim().indexOf(getText().charAt((offset))) < 0)) {
+        token.append(getText().charAt(offset));
+        offset += 1;
+      }
+      TokenAnnotation returnVal = newToken(jcas);
+      // System.out.println("token = " + token.toString() + " fold = " +
+      // foldCase(token.toString()));
+      returnVal.setText(stem(foldCase(token.toString())));
+      returnVal.setBegin(start);
+      returnVal.setEnd(offset);
+
+      return returnVal;
+    }
+    return null;
+  }
+
+
+
+  /**
+   * If one of the case folding flags is true and the input string matches the character pattern
+   * corresponding to that flag, then convert all letters to lowercase.
+   * 
+   * @param token
+   *          The string to case fold
+   * 
+   * @return The case folded string
+   */
+  protected String foldCase(String token) {
+    if (shouldFoldCase(token)) {
+      return doFoldCase(token);
+    }
+    return token;
+  }
+
+  public static String doFoldCase(String token) {
+    return token.trim().toLowerCase();
+  }
+
+
+
+  public boolean shouldFoldCase(String token) {
+    return (caseFoldAll || (caseFoldInitCap && capPat.matcher(token).matches()) || (caseFoldDigit && hasDigit
+            .matcher(token).find()));
+  }
+
+  public boolean shouldStem() {
+    return stemTokens;
+  }
+
+  /**
+   * Set the delimiters used to separate the input string into tokens. This adds the new delimiters
+   * to the base whitespace delimiters " \t\n\r\f".
+   * 
+   * @param delim
+   *          The new set of delimiters.
+   */
+  protected void setDelim(String delim) {
+    this.delim = wsDelim + delim;
+  }
+
+  /**
+   * Set the delimiters used to separate the input string into tokens. This sets the delimiters to
+   * exactly the given set. The base whitespace delimiters are not included.
+   * 
+   * @param delim
+   *          The new set of delimiters.
+   */
+  protected void overrideDelim(String delim) {
+    this.delim = delim;
+  }
+
+  /**
+   * Turn stemming on (true) or off (false).
+   * 
+   * @param flag
+   *          true - turn stemming on, false - turn stemming off
+   */
+  private void setStemming(boolean flag) {
+    stemTokens = flag;
+  }
+
+  /**
+   * Set case folding flag for folding tokens with initial cap.
+   * 
+   * @param flag
+   *          true - turn on case folding, false - turn off case folding.
+   */
+  private void setCaseFoldInitCap(boolean flag) {
+    caseFoldInitCap = flag;
+  }
+
+  /**
+   * Set case folding flag for folding tokens with at least one digit character.
+   * 
+   * @param flag
+   *          true - turn on case folding, false - turn off case folding.
+   */
+  private void setCaseFoldDigit(boolean flag) {
+    caseFoldDigit = flag;
+  }
+
+  /**
+   * Set case folding flag for folding all tokens (subsumes
+   * {@link #setCaseFoldInitCap setCaseFoldInitCap}and {@link #setCaseFoldDigitsetCaseFoldDigit}).
+   * 
+   * @param flag
+   *          true - turn on case folding, false - turn off case folding.
+   */
+  private void setCaseFoldAll(boolean flag) {
+    caseFoldAll = flag;
+  }
+
+  /**
+   * Get the current list of delimiters used to separate the input string into tokens.
+   * 
+   * @return The current list of delimiters used to separate the input string into tokens.
+   */
+  protected String getDelim() {
+    return delim;
+  }
+
+  /**
+   * Get the current stemming flag.
+   * 
+   * @return true if stemming is currently on, false otherwise
+   */
+  protected boolean getStemming() {
+    return stemTokens;
+  }
+
+  /**
+   * Get case folding flag for folding tokens with initial cap.
+   * 
+   * @return the current value of the flag
+   */
+  protected boolean getCaseFoldInitCap() {
+    return (caseFoldInitCap);
+  }
+
+  /**
+   * Get the case folding flag for folding tokens with at least one digit character.
+   * 
+   * @return the current value of the flag
+   */
+  protected boolean getCaseFoldDigit() {
+    return (caseFoldDigit);
+  }
+
+  /**
+   * Get case folding flag for folding all tokens.
+   * 
+   * @return the current value of the flag.
+   */
+  protected boolean getCaseFoldAll() {
+    return (caseFoldAll);
+  }
+
+  /**
+   * Initialize the annotator, which includes compilation of regular expressions, fetching
+   * configuration parameters from XML descriptor file, and loading of the dictionary file.
+   */
+  public void initialize(AnnotatorContext annotatorContext)
+          throws AnnotatorInitializationException, AnnotatorConfigurationException {
+    try {
+      //logger = new Logger("TextTokenizer", annotatorContext.getLogger());
+
+      String[] configParameterNames = annotatorContext.getConfigParameterNames();
+      Object[] configParameters = new Object[configParameterNames.length];
+      for (int i = 0; i < configParameters.length; i++) {
+        configParameters[i] = annotatorContext.getConfigParameterValue(configParameterNames[i]);
+      }
+
+      processAllConfigurationParameters(configParameterNames, configParameters);
+      initTokenizer(configParameterNames, configParameters);
+    } catch (Exception e) {
+      throw new AnnotatorConfigurationException(e);
+    }
+
+  }
+
+
+
+
+
+  public void processAllConfigurationParameters(String[] configParameterNames,
+          Object[] configParameters) throws AnnotatorConfigurationException {
+    for (int i = 0; i < configParameterNames.length; i++) {
+      processConfigurationParameter(configParameterNames[i], configParameters[i]);
+    }
+  }
+
+
+  /**
+   * Perform the actual analysis. Iterate over the document content looking for tokens and post an
+   * annotation for each match found.
+   * 
+   * @param aTCAS
+   *          the current CAS to process.
+   * @param aResultSpec
+   *          a specification of the result annotation that should be created by this annotator
+   * 
+   * @see org.apache.uima.analysis_engine.annotator.JTextAnnotator#process(ResultSpecification)
+   */
+  public void process(JCas jcas, ResultSpecification aResultSpec) throws AnnotatorProcessException {
+
+    try {
+      doTokenization(jcas, jcas.getDocumentText(), getDelim());
+    } catch (Exception e) {
+      throw new AnnotatorProcessException(e);
+    }
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see org.apache.uima.conceptMapper.support.tokenizer.TextTokenizer#initTokenizer(java.lang.String[],
+   *      java.lang.Object[])
+   */
+  public void initTokenizer(String[] paramNames, Object[] paramValues) throws Exception {
+    // nothing to do here
+    return;
+  }
+
+  /**
+   * @param jcas
+   * @param documentText
+   * @param delimiters
+   * @param logger
+   */
+  protected void doTokenization(JCas jcas, String documentText, String delimiters) {
+
+    int numTokens = 0;
+    TokenAnnotation annotation;
+
+    // System.out.println("doTokenization = " + documentText);
+    overrideDelim(delimiters);
+
+    setText(documentText);
+
+    while (null != (annotation = nextToken(jcas))) {
+      annotation.addToIndexes();
+      numTokens++;
+    }
+  }
+
+  /**
+   * @param configParametersNames
+   * @param configParameters
+   */
+  public void processConfigurationParameter(String configParameterName, Object configParameterValue) {
+    if (configParameterName.equals(PARAM_CASE_MATCH)) {
+      String caseSense = (String) configParameterValue;
+      if (caseSense != null) {
+        if (caseSense.equalsIgnoreCase("insensitive")) {
+          this.setCaseFoldInitCap(true);
+        } else if (caseSense.equalsIgnoreCase("digitfold")) {
+          this.setCaseFoldDigit(true);
+        } else if (caseSense.equalsIgnoreCase("ignoreall")) {
+          // System.out.println("SETCASEFOLDALL");
+          this.setCaseFoldAll(true);
+        }
+      }
+    } else if ((configParameterName.equals(PARAM_STEMMER_CLASS)) && (configParameterValue != null)) {
+      try {
+        // System.err.println ("Trying to instantiate stemmer class: '" +
+        // (String) configParameters [i] + "'");
+        Class<?> stemmerClass = Class.forName((String) configParameterValue);
+        setStemmer((Stemmer) stemmerClass.newInstance());
+        setStemming(true);
+      } catch (Exception e) {
+        System.err.println("Exception trying to instantiate stemmer class: '"
+                + (String) configParameterValue + "', original exception:" + e.getMessage());
+        e.printStackTrace();
+      }
+    } else if (configParameterName.equals(PARAM_TOKEN_DELIM)) {
+      String tokenDelimiters = (String) configParameterValue;
+      if (tokenDelimiters != null) {
+        this.setDelim(tokenDelimiters);
+      }
+    }
+  }
+
+
+  /**
+   * If the stemming flag is true, then return the stemmed form of the supplied word using the
+   * Porter stemmer.
+   * 
+   * @param token
+   *          the word to stem
+   * @return the original word if the stemming flag is false, otherwise the stemmed form of the word
+   */
+  protected String stem(String token) {
+    if (shouldStem()) {
+      return doStemming(token, getStemmer());
+    }
+    return token;
+  }
+
+  public static String doStemming(String token, Stemmer stemmer) {
+    return stemmer.stem(token.trim());
+  }
+
+}
\ No newline at end of file

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,115 @@
+
+
+/* First created by JCasGen Tue Mar 25 16:39:54 EDT 2008 */
+package org.apache.uima.conceptMapper.support.tokenizer;
+
+import org.apache.uima.jcas.JCas; 
+import org.apache.uima.jcas.JCasRegistry;
+import org.apache.uima.jcas.cas.TOP_Type;
+
+
+
+/** offset tokenizer functionality
+ * Updated by JCasGen Tue Mar 25 16:39:54 EDT 2008
+ * XML source: /OtherStuff/IBM/eclipse-apacheuima/conceptMapper/src/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.xml
+ * @generated */
+public class TokenAnnotation extends uima.tt.TokenAnnotation {
+  /** @generated
+   * @ordered 
+   */
+  public final static int typeIndexID = JCasRegistry.register(TokenAnnotation.class);
+  /** @generated
+   * @ordered 
+   */
+  public final static int type = typeIndexID;
+  /** @generated  */
+  public              int getTypeIndexID() {return typeIndexID;}
+ 
+  /** Never called.  Disable default constructor
+   * @generated */
+  protected TokenAnnotation() {}
+    
+  /** Internal - constructor used by generator 
+   * @generated */
+  public TokenAnnotation(int addr, TOP_Type type) {
+    super(addr, type);
+    readObject();
+  }
+  
+  /** @generated */
+  public TokenAnnotation(JCas jcas) {
+    super(jcas);
+    readObject();   
+  } 
+
+  /** @generated */  
+  public TokenAnnotation(JCas jcas, int begin, int end) {
+    super(jcas);
+    setBegin(begin);
+    setEnd(end);
+    readObject();
+  }   
+
+  /** <!-- begin-user-doc -->
+    * Write your own initialization here
+    * <!-- end-user-doc -->
+  @generated modifiable */
+  private void readObject() {}
+     
+ 
+    
+  //*--------------*
+  //* Feature: text
+
+  /** getter for text - gets text of token
+   * @generated */
+  public String getText() {
+    if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_text == null)
+      jcasType.jcas.throwFeatMissing("text", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    return jcasType.ll_cas.ll_getStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_text);}
+    
+  /** setter for text - sets text of token 
+   * @generated */
+  public void setText(String v) {
+    if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_text == null)
+      jcasType.jcas.throwFeatMissing("text", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    jcasType.ll_cas.ll_setStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_text, v);}    
+   
+    
+  //*--------------*
+  //* Feature: tokenType
+
+  /** getter for tokenType - gets 
+   * @generated */
+  public int getTokenType() {
+    if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenType == null)
+      jcasType.jcas.throwFeatMissing("tokenType", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    return jcasType.ll_cas.ll_getIntValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenType);}
+    
+  /** setter for tokenType - sets  
+   * @generated */
+  public void setTokenType(int v) {
+    if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenType == null)
+      jcasType.jcas.throwFeatMissing("tokenType", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    jcasType.ll_cas.ll_setIntValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenType, v);}    
+   
+    
+  //*--------------*
+  //* Feature: tokenClass
+
+  /** getter for tokenClass - gets semantic class, or other such classification of this token
+   * @generated */
+  public String getTokenClass() {
+    if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenClass == null)
+      jcasType.jcas.throwFeatMissing("tokenClass", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    return jcasType.ll_cas.ll_getStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenClass);}
+    
+  /** setter for tokenClass - sets semantic class, or other such classification of this token 
+   * @generated */
+  public void setTokenClass(String v) {
+    if (TokenAnnotation_Type.featOkTst && ((TokenAnnotation_Type)jcasType).casFeat_tokenClass == null)
+      jcasType.jcas.throwFeatMissing("tokenClass", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    jcasType.ll_cas.ll_setStringValue(addr, ((TokenAnnotation_Type)jcasType).casFeatCode_tokenClass, v);}    
+  }
+
+    
\ No newline at end of file

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.xml
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.xml?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.xml (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation.xml Mon Jun 23 05:31:40 2008
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+	Licensed to the Apache Software Foundation (ASF) under one
+	or more contributor license agreements.  See the NOTICE file
+	distributed with this work for additional information
+	regarding copyright ownership.  The ASF licenses this file
+	to you under the Apache License, Version 2.0 (the
+	"License"); you may not use this file except in compliance
+	with the License.  You may obtain a copy of the License at
+	
+	http://www.apache.org/licenses/LICENSE-2.0
+	
+	Unless required by applicable law or agreed to in writing,
+	software distributed under the License is distributed on an
+	"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+	KIND, either express or implied.  See the License for the
+	specific language governing permissions and limitations
+	under the License.    
+-->
+<typeSystemDescription xmlns="http://uima.apache.org/resourceSpecifier">
+<name>TokenAnnotation</name>
+<description>offset token type for CAS</description>
+<version>1.0</version>
+<vendor></vendor>
+<types>
+<typeDescription>
+<name>uima.tt.TokenAnnotation</name>
+<description></description>
+<supertypeName>uima.tcas.Annotation</supertypeName>
+</typeDescription>
+<typeDescription>
+<name>org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation</name>
+<description>offset tokenizer functionality</description>
+<supertypeName>uima.tt.TokenAnnotation</supertypeName>
+<features>
+<featureDescription>
+<name>text</name>
+<description>text of token</description>
+<rangeTypeName>uima.cas.String</rangeTypeName>
+</featureDescription>
+<featureDescription>
+<name>tokenType</name>
+<description></description>
+<rangeTypeName>uima.cas.Integer</rangeTypeName>
+</featureDescription>
+<featureDescription>
+<name>tokenClass</name>
+<description>semantic class, or other such classification of this token</description>
+<rangeTypeName>uima.cas.String</rangeTypeName>
+</featureDescription>
+</features>
+</typeDescription>
+</types>
+</typeSystemDescription>

Added: incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation_Type.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation_Type.java?rev=670541&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation_Type.java (added)
+++ incubator/uima/sandbox/trunk/ConceptMapper/src/main/java/org/apache/uima/conceptMapper/support/tokenizer/TokenAnnotation_Type.java Mon Jun 23 05:31:40 2008
@@ -0,0 +1,122 @@
+
+/* First created by JCasGen Tue Mar 25 16:39:54 EDT 2008 */
+package org.apache.uima.conceptMapper.support.tokenizer;
+
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.JCasRegistry;
+import org.apache.uima.cas.impl.CASImpl;
+import org.apache.uima.cas.impl.FSGenerator;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.impl.TypeImpl;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.impl.FeatureImpl;
+import org.apache.uima.cas.Feature;
+
+/** offset tokenizer functionality
+ * Updated by JCasGen Tue Mar 25 16:39:54 EDT 2008
+ * @generated */
+public class TokenAnnotation_Type extends uima.tt.TokenAnnotation_Type {
+  /** @generated */
+  protected FSGenerator getFSGenerator() {return fsGenerator;}
+  /** @generated */
+  private final FSGenerator fsGenerator = 
+    new FSGenerator() {
+      public FeatureStructure createFS(int addr, CASImpl cas) {
+  			 if (TokenAnnotation_Type.this.useExistingInstance) {
+  			   // Return eq fs instance if already created
+  		     FeatureStructure fs = TokenAnnotation_Type.this.jcas.getJfsFromCaddr(addr);
+  		     if (null == fs) {
+  		       fs = new TokenAnnotation(addr, TokenAnnotation_Type.this);
+  			   TokenAnnotation_Type.this.jcas.putJfsFromCaddr(addr, fs);
+  			   return fs;
+  		     }
+  		     return fs;
+        } else return new TokenAnnotation(addr, TokenAnnotation_Type.this);
+  	  }
+    };
+  /** @generated */
+  public final static int typeIndexID = TokenAnnotation.typeIndexID;
+  /** @generated 
+     @modifiable */
+  public final static boolean featOkTst = JCasRegistry.getFeatOkTst("org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+ 
+  /** @generated */
+  final Feature casFeat_text;
+  /** @generated */
+  final int     casFeatCode_text;
+  /** @generated */ 
+  public String getText(int addr) {
+        if (featOkTst && casFeat_text == null)
+      jcas.throwFeatMissing("text", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    return ll_cas.ll_getStringValue(addr, casFeatCode_text);
+  }
+  /** @generated */    
+  public void setText(int addr, String v) {
+        if (featOkTst && casFeat_text == null)
+      jcas.throwFeatMissing("text", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    ll_cas.ll_setStringValue(addr, casFeatCode_text, v);}
+    
+  
+ 
+  /** @generated */
+  final Feature casFeat_tokenType;
+  /** @generated */
+  final int     casFeatCode_tokenType;
+  /** @generated */ 
+  public int getTokenType(int addr) {
+        if (featOkTst && casFeat_tokenType == null)
+      jcas.throwFeatMissing("tokenType", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    return ll_cas.ll_getIntValue(addr, casFeatCode_tokenType);
+  }
+  /** @generated */    
+  public void setTokenType(int addr, int v) {
+        if (featOkTst && casFeat_tokenType == null)
+      jcas.throwFeatMissing("tokenType", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    ll_cas.ll_setIntValue(addr, casFeatCode_tokenType, v);}
+    
+  
+ 
+  /** @generated */
+  final Feature casFeat_tokenClass;
+  /** @generated */
+  final int     casFeatCode_tokenClass;
+  /** @generated */ 
+  public String getTokenClass(int addr) {
+        if (featOkTst && casFeat_tokenClass == null)
+      jcas.throwFeatMissing("tokenClass", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    return ll_cas.ll_getStringValue(addr, casFeatCode_tokenClass);
+  }
+  /** @generated */    
+  public void setTokenClass(int addr, String v) {
+        if (featOkTst && casFeat_tokenClass == null)
+      jcas.throwFeatMissing("tokenClass", "org.apache.uima.conceptMapper.support.tokenizer.TokenAnnotation");
+    ll_cas.ll_setStringValue(addr, casFeatCode_tokenClass, v);}
+    
+  
+
+
+
+  /** initialize variables to correspond with Cas Type and Features
+	* @generated */
+  public TokenAnnotation_Type(JCas jcas, Type casType) {
+    super(jcas, casType);
+    casImpl.getFSClassRegistry().addGeneratorForType((TypeImpl)this.casType, getFSGenerator());
+
+ 
+    casFeat_text = jcas.getRequiredFeatureDE(casType, "text", "uima.cas.String", featOkTst);
+    casFeatCode_text  = (null == casFeat_text) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_text).getCode();
+
+ 
+    casFeat_tokenType = jcas.getRequiredFeatureDE(casType, "tokenType", "uima.cas.Integer", featOkTst);
+    casFeatCode_tokenType  = (null == casFeat_tokenType) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_tokenType).getCode();
+
+ 
+    casFeat_tokenClass = jcas.getRequiredFeatureDE(casType, "tokenClass", "uima.cas.String", featOkTst);
+    casFeatCode_tokenClass  = (null == casFeat_tokenClass) ? JCas.INVALID_FEATURE_CODE : ((FeatureImpl)casFeat_tokenClass).getCode();
+
+  }
+}
+
+
+
+    
\ No newline at end of file