You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ctakes.apache.org by ja...@apache.org on 2012/10/31 06:26:55 UTC
svn commit: r1403989 [6/28] - in /incubator/ctakes/branches/SHARPn-cTAKES:
Constituency Parser/src/org/chboston/cnlp/ctakes/parser/ Constituency
Parser/src/org/chboston/cnlp/ctakes/parser/uima/ae/ Constituency
Parser/src/org/chboston/cnlp/ctakes/parser...
Modified: incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/src/edu/mayo/bmi/uima/pos_tagger/TagDictionaryCreator.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/POS%20tagger/src/edu/mayo/bmi/uima/pos_tagger/TagDictionaryCreator.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/src/edu/mayo/bmi/uima/pos_tagger/TagDictionaryCreator.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/src/edu/mayo/bmi/uima/pos_tagger/TagDictionaryCreator.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2009 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -21,176 +14,176 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.mayo.bmi.uima.pos_tagger;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.Reader;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * From a Part of Speech (POS) corpus in OpenNLP format, create a tagDictionary
- * <br>Outputs the dictionary to the specified file
- * @author Mayo Clinic
- */
-
-public class TagDictionaryCreator {
-
- /*
- * Writes the dictionary entries to a PrintStream
- * The tags are sorted for each entry, to make it easier to read, and also to make
- * unit testing simpler
- */
- private static void writeDictionary(Map<String, Set<String>> dict, PrintStream out) throws IOException {
- String line;
- for (Object key : dict.keySet()) {
- line = key.toString();
- // line = key.toString() + " " + dict.get(key);
- //ArrayList<String> sorted = new ArrayList<String>(dict.get(key).size());
- String[] sorted = new String[dict.get(key).size()];
- sorted = dict.get(key).toArray(sorted);
- if (sorted!=null) java.util.Arrays.sort(sorted);
- // sorted =
- for (String s : sorted) {
- line = line + " " + s;
- }
- out.println(line); // output the dictionary entry to the PrintStream
- }
- }
-
- /**
- * Create a tag dictionary from a POS corpus in OpenNLP format.
- * @param br
- * @param caseSensitive
- * @return
- * @throws IOException
- */
- public static HashMap<String, Set<String>> createTagDictionary(BufferedReader br, boolean caseSensitive) throws IOException {
- HashMap<String, Set<String>> dict = new HashMap<String, Set<String>> (50000); // initial size is arbitrary
- String line;
- String word;
- String tag;
- int position; // position of last underscore
- Set<String> tagSet;
- while((line = br.readLine()) != null) {
- for (String token : line.split(" ")) {
- position = token.lastIndexOf('_');
- if (position == -1 || position == token.length()-1) {
- System.err.println("WARNING: '" + token + "' does not conform to the format WORD_TAG");
- continue;
- }
- word = token.substring(0, position);
- if(!caseSensitive)
- word = word.toLowerCase();
- tag = token.substring(position+1);
- tagSet = dict.get(word);
- if (tagSet==null) {
- tagSet = new HashSet<String>();
- tagSet.add(tag);
- dict.put(word, tagSet);
- }
- else {
- if (!tagSet.contains(tag)) {
- tagSet.add(tag);
- }
- else {
- }
- }
- }
- }
- return dict;
- }
-
- private static BufferedReader getBufferedReader(String filename) throws FileNotFoundException {
- File f = new File(filename);
- Reader r;
- try {
- r = new FileReader(f);
- } catch (FileNotFoundException e) {
- System.err.println("Error reading from file " + filename);
- throw e;
- }
-
- return new BufferedReader(r);
- }
-
-
- /**
- * Read a file containing POS-tagged tokens in OpenNLP format,
- * and output a tagDictionary (OpenNLP format)<br>
- * Example input:
- * <br>winning_JJ body_NN
- * <br>winning_VBG<br>
- * Example output:
- * <br>body NN
- * <br>winning JJ VBG
- *
- * @see #printUsage()
- */
- public static void main(String[] args) {
-
- boolean argumentsCorrect = true;
- if(args == null || args.length != 3)
- argumentsCorrect = false;
- for(String arg : args) {
- if(arg == null || arg.trim().length()==0)
- argumentsCorrect = false;
- }
- if (!argumentsCorrect) {
- System.err.println("ERROR: three non-empty arguments are required.");
- printUsage();
- return;
- }
-
- String arg0 = args[0].trim();
- if (arg0.equals("-h") || (arg0.equals("--help"))) {
- printUsage();
- return;
- }
-
- String trainingDataFn = args[0];
- String tagDictFn = args[1];
- String caseSensitiveArg = args[2];
- try {
-
- PrintStream out = new PrintStream(tagDictFn);
- boolean caseSensitive = Boolean.parseBoolean(caseSensitiveArg);
- Map<String, Set<String>> tagDictionary;
-
- BufferedReader br = getBufferedReader(trainingDataFn);
- tagDictionary = createTagDictionary(br, caseSensitive);
- writeDictionary(tagDictionary, out);
- out.flush();
- out.close();
- System.out.println("TagDictionary written to " + tagDictFn);
-
- } catch (IOException e) {
- System.err.println("TagDictionaryCreator Failed");
- System.err.println("training-data = " + trainingDataFn);
- System.err.println("tag-dictionary = " + tagDictFn);
- System.err.println("case-sensitive = " + caseSensitiveArg);
- System.err.flush();
- printUsage();
- File f = new File(args[0]);
- System.err.println("training-data absolute path = " + f.getAbsolutePath());
- }
-
- }
-
- public static void printUsage() {
-
- System.out.println("Usage: java TagDictionaryCreator <training-data> <tag-dictionary> <case-sensitive>");
- System.out.println(" where <training-data> is a file prepared for training the part-of-speech tagger as described in data/pos/training/README");
- System.out.println(" where <tag-dictionary> is the output file where the tag dictionary will be written");
- System.out.println(" where <case-sensitive> is either 'true' or 'false' depending on whether the tag dictionary should be case sensitive or not.");
-
- }
-
-}
+package edu.mayo.bmi.uima.pos_tagger;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * From a Part of Speech (POS) corpus in OpenNLP format, create a tagDictionary
+ * <br>Outputs the dictionary to the specified file
+ * @author Mayo Clinic
+ */
+
+public class TagDictionaryCreator {
+
+ /*
+ * Writes the dictionary entries to a PrintStream
+ * The tags are sorted for each entry, to make it easier to read, and also to make
+ * unit testing simpler
+ */
+ private static void writeDictionary(Map<String, Set<String>> dict, PrintStream out) throws IOException {
+ String line;
+ for (Object key : dict.keySet()) {
+ line = key.toString();
+ // line = key.toString() + " " + dict.get(key);
+ //ArrayList<String> sorted = new ArrayList<String>(dict.get(key).size());
+ String[] sorted = new String[dict.get(key).size()];
+ sorted = dict.get(key).toArray(sorted);
+ if (sorted!=null) java.util.Arrays.sort(sorted);
+ // sorted =
+ for (String s : sorted) {
+ line = line + " " + s;
+ }
+ out.println(line); // output the dictionary entry to the PrintStream
+ }
+ }
+
+ /**
+ * Create a tag dictionary from a POS corpus in OpenNLP format.
+ * @param br
+ * @param caseSensitive
+ * @return
+ * @throws IOException
+ */
+ public static HashMap<String, Set<String>> createTagDictionary(BufferedReader br, boolean caseSensitive) throws IOException {
+ HashMap<String, Set<String>> dict = new HashMap<String, Set<String>> (50000); // initial size is arbitrary
+ String line;
+ String word;
+ String tag;
+ int position; // position of last underscore
+ Set<String> tagSet;
+ while((line = br.readLine()) != null) {
+ for (String token : line.split(" ")) {
+ position = token.lastIndexOf('_');
+ if (position == -1 || position == token.length()-1) {
+ System.err.println("WARNING: '" + token + "' does not conform to the format WORD_TAG");
+ continue;
+ }
+ word = token.substring(0, position);
+ if(!caseSensitive)
+ word = word.toLowerCase();
+ tag = token.substring(position+1);
+ tagSet = dict.get(word);
+ if (tagSet==null) {
+ tagSet = new HashSet<String>();
+ tagSet.add(tag);
+ dict.put(word, tagSet);
+ }
+ else {
+ if (!tagSet.contains(tag)) {
+ tagSet.add(tag);
+ }
+ else {
+ }
+ }
+ }
+ }
+ return dict;
+ }
+
+ private static BufferedReader getBufferedReader(String filename) throws FileNotFoundException {
+ File f = new File(filename);
+ Reader r;
+ try {
+ r = new FileReader(f);
+ } catch (FileNotFoundException e) {
+ System.err.println("Error reading from file " + filename);
+ throw e;
+ }
+
+ return new BufferedReader(r);
+ }
+
+
+ /**
+ * Read a file containing POS-tagged tokens in OpenNLP format,
+ * and output a tagDictionary (OpenNLP format)<br>
+ * Example input:
+ * <br>winning_JJ body_NN
+ * <br>winning_VBG<br>
+ * Example output:
+ * <br>body NN
+ * <br>winning JJ VBG
+ *
+ * @see #printUsage()
+ */
+ public static void main(String[] args) {
+
+ boolean argumentsCorrect = true;
+ if(args == null || args.length != 3)
+ argumentsCorrect = false;
+ for(String arg : args) {
+ if(arg == null || arg.trim().length()==0)
+ argumentsCorrect = false;
+ }
+ if (!argumentsCorrect) {
+ System.err.println("ERROR: three non-empty arguments are required.");
+ printUsage();
+ return;
+ }
+
+ String arg0 = args[0].trim();
+ if (arg0.equals("-h") || (arg0.equals("--help"))) {
+ printUsage();
+ return;
+ }
+
+ String trainingDataFn = args[0];
+ String tagDictFn = args[1];
+ String caseSensitiveArg = args[2];
+ try {
+
+ PrintStream out = new PrintStream(tagDictFn);
+ boolean caseSensitive = Boolean.parseBoolean(caseSensitiveArg);
+ Map<String, Set<String>> tagDictionary;
+
+ BufferedReader br = getBufferedReader(trainingDataFn);
+ tagDictionary = createTagDictionary(br, caseSensitive);
+ writeDictionary(tagDictionary, out);
+ out.flush();
+ out.close();
+ System.out.println("TagDictionary written to " + tagDictFn);
+
+ } catch (IOException e) {
+ System.err.println("TagDictionaryCreator Failed");
+ System.err.println("training-data = " + trainingDataFn);
+ System.err.println("tag-dictionary = " + tagDictFn);
+ System.err.println("case-sensitive = " + caseSensitiveArg);
+ System.err.flush();
+ printUsage();
+ File f = new File(args[0]);
+ System.err.println("training-data absolute path = " + f.getAbsolutePath());
+ }
+
+ }
+
+ public static void printUsage() {
+
+ System.out.println("Usage: java TagDictionaryCreator <training-data> <tag-dictionary> <case-sensitive>");
+ System.out.println(" where <training-data> is a file prepared for training the part-of-speech tagger as described in data/pos/training/README");
+ System.out.println(" where <tag-dictionary> is the output file where the tag dictionary will be written");
+ System.out.println(" where <case-sensitive> is either 'true' or 'false' depending on whether the tag dictionary should be case sensitive or not.");
+
+ }
+
+}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/CpeTests.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/POS%20tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/CpeTests.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/CpeTests.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/CpeTests.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2009 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -21,240 +14,240 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.mayo.bmi.uima.pos_tagger.test;
-
-
-// from org.apache.uima.examples.cpe.CpeTests.java in the %UIMA_HOME%\examples\src
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.util.List;
-
-import org.apache.uima.UIMAFramework;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.collection.CollectionProcessingEngine;
-import org.apache.uima.collection.EntityProcessStatus;
-import org.apache.uima.collection.StatusCallbackListener;
-import org.apache.uima.collection.metadata.CpeDescription;
-import org.apache.uima.util.XMLInputSource;
-
-/**
- * Main Class that runs a Collection Processing Engine (CPE). This class reads a CPE Descriptor as a
- * command-line argument and instantiates the CPE. It also registers a callback listener with the
- * CPE, which will print progress and statistics to System.out.
- *
- *
- */
-public class CpeTests extends Thread {
- /**
- * The CPE instance.
- */
- private CollectionProcessingEngine mCPE;
-
- /**
- * Start time of CPE initialization
- */
- private long mStartTime;
-
- /**
- * Start time of the processing
- */
- private long mInitCompleteTime;
-
- /**
- * Constructor for the class.
- *
- * @param args
- * command line arguments into the program - see class description
- */
- public CpeTests(String args[]) throws Exception {
- mStartTime = System.currentTimeMillis();
-
- // check command line args
- if (args.length < 1) {
- printUsageMessage();
- System.exit(1);
- }
-
- // parse CPE descriptor
- System.out.println("Parsing CPE Descriptor");
- CpeDescription cpeDesc = UIMAFramework.getXMLParser().parseCpeDescription(
- new XMLInputSource(args[0]));
- // instantiate CPE
- System.out.println("Instantiating CPE");
- mCPE = UIMAFramework.produceCollectionProcessingEngine(cpeDesc);
-
- // Create and register a Status Callback Listener
- mCPE.addStatusCallbackListener(new StatusCallbackListenerImpl());
-
- // Start Processing
- System.out.println("Running CPE");
- mCPE.process();
-
- // Allow user to abort by pressing Enter
- System.out.println("To abort processing, type \"abort\" and press enter.");
- while (true) {
- String line = new BufferedReader(new InputStreamReader(System.in)).readLine();
- if ("abort".equals(line) && mCPE.isProcessing()) {
- System.out.println("Aborting...");
- mCPE.stop();
- break;
- }
- }
- }
-
- /**
- *
- */
- private static void printUsageMessage() {
- System.out.println(" Arguments to the program are as follows : \n"
- + "args[0] : path to CPE descriptor file");
- }
-
- /**
- * main class.
- *
- * @param args
- * Command line arguments - see class description
- */
- public static void main(String[] args) throws Exception {
- new CpeTests(args);
- }
-
- /**
- * Callback Listener. Receives event notifications from CPE.
- *
- *
- */
- class StatusCallbackListenerImpl implements StatusCallbackListener {
- int entityCount = 0;
-
- long size = 0;
-
- /**
- * Called when the initialization is completed.
- *
- * @see org.apache.uima.collection.processing.StatusCallbackListener#initializationComplete()
- */
- public void initializationComplete() {
- System.out.println("CPM Initialization Complete");
- mInitCompleteTime = System.currentTimeMillis();
- }
-
- /**
- * Called when the batchProcessing is completed.
- *
- * @see org.apache.uima.collection.processing.StatusCallbackListener#batchProcessComplete()
- *
- */
- public void batchProcessComplete() {
- System.out.print("Completed " + entityCount + " documents");
- if (size > 0) {
- System.out.print("; " + size + " characters");
- }
- System.out.println();
- long elapsedTime = System.currentTimeMillis() - mStartTime;
- System.out.println("Time Elapsed : " + elapsedTime + " ms ");
- }
-
- /**
- * Called when the collection processing is completed.
- *
- * @see org.apache.uima.collection.processing.StatusCallbackListener#collectionProcessComplete()
- */
- public void collectionProcessComplete() {
- long time = System.currentTimeMillis();
- System.out.print("Completed " + entityCount + " documents");
- if (size > 0) {
- System.out.print("; " + size + " characters");
- }
- System.out.println();
- long initTime = mInitCompleteTime - mStartTime;
- long processingTime = time - mInitCompleteTime;
- long elapsedTime = initTime + processingTime;
- System.out.println("Total Time Elapsed: " + elapsedTime + " ms ");
- System.out.println("Initialization Time: " + initTime + " ms");
- System.out.println("Processing Time: " + processingTime + " ms");
-
- System.out.println("\n\n ------------------ PERFORMANCE REPORT ------------------\n");
- System.out.println(mCPE.getPerformanceReport().toString());
- // stop the JVM. Otherwise main thread will still be blocked waiting for
- // user to press Enter.
- System.exit(1);
- }
-
- /**
- * Called when the CPM is paused.
- *
- * @see org.apache.uima.collection.processing.StatusCallbackListener#paused()
- */
- public void paused() {
- System.out.println("Paused");
- }
-
- /**
- * Called when the CPM is resumed after a pause.
- *
- * @see org.apache.uima.collection.processing.StatusCallbackListener#resumed()
- */
- public void resumed() {
- System.out.println("Resumed");
- }
-
- /**
- * Called when the CPM is stopped abruptly due to errors.
- *
- * @see org.apache.uima.collection.processing.StatusCallbackListener#aborted()
- */
- public void aborted() {
- System.out.println("Aborted");
- // stop the JVM. Otherwise main thread will still be blocked waiting for
- // user to press Enter.
- System.exit(1);
- }
-
- /**
- * Called when the processing of a Document is completed. <br>
- * The process status can be looked at and corresponding actions taken.
- *
- * @param aCas
- * CAS corresponding to the completed processing
- * @param aStatus
- * EntityProcessStatus that holds the status of all the events for aEntity
- */
- public void entityProcessComplete(CAS aCas, EntityProcessStatus aStatus) {
- if (aStatus.isException()) {
- List<?> exceptions = aStatus.getExceptions();
- for (int i = 0; i < exceptions.size(); i++) {
- ((Throwable) exceptions.get(i)).printStackTrace();
- }
- return;
- }
- entityCount++;
- String docText = aCas.getDocumentText();
- if (docText != null) {
- size += docText.length();
- }
- }
- }
-
-}
+package edu.mayo.bmi.uima.pos_tagger.test;
+
+
+// from org.apache.uima.examples.cpe.CpeTests.java in the %UIMA_HOME%\examples\src
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.List;
+
+import org.apache.uima.UIMAFramework;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.collection.CollectionProcessingEngine;
+import org.apache.uima.collection.EntityProcessStatus;
+import org.apache.uima.collection.StatusCallbackListener;
+import org.apache.uima.collection.metadata.CpeDescription;
+import org.apache.uima.util.XMLInputSource;
+
+/**
+ * Main Class that runs a Collection Processing Engine (CPE). This class reads a CPE Descriptor as a
+ * command-line argument and instantiates the CPE. It also registers a callback listener with the
+ * CPE, which will print progress and statistics to System.out.
+ *
+ *
+ */
+public class CpeTests extends Thread {
+ /**
+ * The CPE instance.
+ */
+ private CollectionProcessingEngine mCPE;
+
+ /**
+ * Start time of CPE initialization
+ */
+ private long mStartTime;
+
+ /**
+ * Start time of the processing
+ */
+ private long mInitCompleteTime;
+
+ /**
+ * Constructor for the class.
+ *
+ * @param args
+ * command line arguments into the program - see class description
+ */
+ public CpeTests(String args[]) throws Exception {
+ mStartTime = System.currentTimeMillis();
+
+ // check command line args
+ if (args.length < 1) {
+ printUsageMessage();
+ System.exit(1);
+ }
+
+ // parse CPE descriptor
+ System.out.println("Parsing CPE Descriptor");
+ CpeDescription cpeDesc = UIMAFramework.getXMLParser().parseCpeDescription(
+ new XMLInputSource(args[0]));
+ // instantiate CPE
+ System.out.println("Instantiating CPE");
+ mCPE = UIMAFramework.produceCollectionProcessingEngine(cpeDesc);
+
+ // Create and register a Status Callback Listener
+ mCPE.addStatusCallbackListener(new StatusCallbackListenerImpl());
+
+ // Start Processing
+ System.out.println("Running CPE");
+ mCPE.process();
+
+ // Allow user to abort by pressing Enter
+ System.out.println("To abort processing, type \"abort\" and press enter.");
+ while (true) {
+ String line = new BufferedReader(new InputStreamReader(System.in)).readLine();
+ if ("abort".equals(line) && mCPE.isProcessing()) {
+ System.out.println("Aborting...");
+ mCPE.stop();
+ break;
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static void printUsageMessage() {
+ System.out.println(" Arguments to the program are as follows : \n"
+ + "args[0] : path to CPE descriptor file");
+ }
+
+ /**
+ * main class.
+ *
+ * @param args
+ * Command line arguments - see class description
+ */
+ public static void main(String[] args) throws Exception {
+ new CpeTests(args);
+ }
+
+ /**
+ * Callback Listener. Receives event notifications from CPE.
+ *
+ *
+ */
+ class StatusCallbackListenerImpl implements StatusCallbackListener {
+ int entityCount = 0;
+
+ long size = 0;
+
+ /**
+ * Called when the initialization is completed.
+ *
+ * @see org.apache.uima.collection.processing.StatusCallbackListener#initializationComplete()
+ */
+ public void initializationComplete() {
+ System.out.println("CPM Initialization Complete");
+ mInitCompleteTime = System.currentTimeMillis();
+ }
+
+ /**
+ * Called when the batchProcessing is completed.
+ *
+ * @see org.apache.uima.collection.processing.StatusCallbackListener#batchProcessComplete()
+ *
+ */
+ public void batchProcessComplete() {
+ System.out.print("Completed " + entityCount + " documents");
+ if (size > 0) {
+ System.out.print("; " + size + " characters");
+ }
+ System.out.println();
+ long elapsedTime = System.currentTimeMillis() - mStartTime;
+ System.out.println("Time Elapsed : " + elapsedTime + " ms ");
+ }
+
+ /**
+ * Called when the collection processing is completed.
+ *
+ * @see org.apache.uima.collection.processing.StatusCallbackListener#collectionProcessComplete()
+ */
+ public void collectionProcessComplete() {
+ long time = System.currentTimeMillis();
+ System.out.print("Completed " + entityCount + " documents");
+ if (size > 0) {
+ System.out.print("; " + size + " characters");
+ }
+ System.out.println();
+ long initTime = mInitCompleteTime - mStartTime;
+ long processingTime = time - mInitCompleteTime;
+ long elapsedTime = initTime + processingTime;
+ System.out.println("Total Time Elapsed: " + elapsedTime + " ms ");
+ System.out.println("Initialization Time: " + initTime + " ms");
+ System.out.println("Processing Time: " + processingTime + " ms");
+
+ System.out.println("\n\n ------------------ PERFORMANCE REPORT ------------------\n");
+ System.out.println(mCPE.getPerformanceReport().toString());
+ // stop the JVM. Otherwise main thread will still be blocked waiting for
+ // user to press Enter.
+ System.exit(1);
+ }
+
+ /**
+ * Called when the CPM is paused.
+ *
+ * @see org.apache.uima.collection.processing.StatusCallbackListener#paused()
+ */
+ public void paused() {
+ System.out.println("Paused");
+ }
+
+ /**
+ * Called when the CPM is resumed after a pause.
+ *
+ * @see org.apache.uima.collection.processing.StatusCallbackListener#resumed()
+ */
+ public void resumed() {
+ System.out.println("Resumed");
+ }
+
+ /**
+ * Called when the CPM is stopped abruptly due to errors.
+ *
+ * @see org.apache.uima.collection.processing.StatusCallbackListener#aborted()
+ */
+ public void aborted() {
+ System.out.println("Aborted");
+ // stop the JVM. Otherwise main thread will still be blocked waiting for
+ // user to press Enter.
+ System.exit(1);
+ }
+
+ /**
+ * Called when the processing of a Document is completed. <br>
+ * The process status can be looked at and corresponding actions taken.
+ *
+ * @param aCas
+ * CAS corresponding to the completed processing
+ * @param aStatus
+ * EntityProcessStatus that holds the status of all the events for aEntity
+ */
+ public void entityProcessComplete(CAS aCas, EntityProcessStatus aStatus) {
+ if (aStatus.isException()) {
+ List<?> exceptions = aStatus.getExceptions();
+ for (int i = 0; i < exceptions.size(); i++) {
+ ((Throwable) exceptions.get(i)).printStackTrace();
+ }
+ return;
+ }
+ entityCount++;
+ String docText = aCas.getDocumentText();
+ if (docText != null) {
+ size += docText.length();
+ }
+ }
+ }
+
+}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/OpenNLPPOSCollectionReaderTests.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/POS%20tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/OpenNLPPOSCollectionReaderTests.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/OpenNLPPOSCollectionReaderTests.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/OpenNLPPOSCollectionReaderTests.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2009 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -21,143 +14,143 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.mayo.bmi.uima.pos_tagger.test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.uima.analysis_engine.AnalysisEngine;
-import org.apache.uima.collection.CollectionException;
-import org.apache.uima.collection.CollectionReader;
-import org.apache.uima.jcas.JCas;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.junit.Test;
-
-import edu.mayo.bmi.uima.core.test.TestUtil;
-import edu.mayo.bmi.uima.core.type.syntax.BaseToken;
-
-public class OpenNLPPOSCollectionReaderTests {
-
- @Test
- public void testReader() throws ResourceInitializationException, IOException, CollectionException {
- CollectionReader collectionReader = TestUtil.getCR(new File("test/desc/OpenNLPPOSCollectionReader.xml"));
- AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/NullAnnotator.xml"));
- JCas jCas = analysisEngine.newJCas();
- collectionReader.getNext(jCas.getCas());
-
- BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals(0, baseToken.getBegin());
- assertEquals(1, baseToken.getEnd());
- assertEquals("A", baseToken.getCoveredText());
- assertEquals("A", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals("farmer", baseToken.getCoveredText());
- assertEquals("B", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("went", baseToken.getCoveredText());
- assertEquals("CC", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("trotting", baseToken.getCoveredText());
- assertEquals("DDD", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
- assertEquals("upon_A", baseToken.getCoveredText());
- assertEquals("E", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
- assertEquals("his", baseToken.getCoveredText());
- assertEquals("EE", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
- assertEquals(".", baseToken.getCoveredText());
- assertEquals(".", baseToken.getPartOfSpeech());
-
- CollectionException ce = null;
- jCas = analysisEngine.newJCas();
- try {
- collectionReader.getNext(jCas.getCas());
- } catch(CollectionException e) {
- ce = e;
- }
- assertNotNull(ce);
-
- jCas = analysisEngine.newJCas();
- collectionReader.getNext(jCas.getCas());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals("A_", baseToken.getCoveredText());
- assertEquals("A", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals("_", baseToken.getCoveredText());
- assertEquals("B", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("_B_", baseToken.getCoveredText());
- assertEquals("C", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("B", baseToken.getCoveredText());
- assertEquals("_", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
- assertEquals("__", baseToken.getCoveredText());
- assertEquals("_", baseToken.getPartOfSpeech());
- }
-
- @Test
- public void testLoadWordsOnly() throws ResourceInitializationException, IOException, CollectionException {
- CollectionReader collectionReader = TestUtil.getCR(new File("test/desc/OpenNLPPOSCollectionReader2.xml"));
- AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/NullAnnotator.xml"));
- JCas jCas = analysisEngine.newJCas();
- collectionReader.getNext(jCas.getCas());
-
- BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals(0, baseToken.getBegin());
- assertEquals(1, baseToken.getEnd());
- assertEquals("A", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals("farmer", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("went", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("trotting", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
- assertEquals("upon_A", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
- assertEquals("his", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
- assertEquals(".", baseToken.getCoveredText());
- assertNull(baseToken.getPartOfSpeech());
-
- CollectionException ce = null;
- jCas = analysisEngine.newJCas();
- try {
- collectionReader.getNext(jCas.getCas());
- } catch(CollectionException e) {
- ce = e;
- }
- assertNotNull(ce);
-
- }
-
-}
+package edu.mayo.bmi.uima.pos_tagger.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.uima.analysis_engine.AnalysisEngine;
+import org.apache.uima.collection.CollectionException;
+import org.apache.uima.collection.CollectionReader;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.junit.Test;
+
+import edu.mayo.bmi.uima.core.test.TestUtil;
+import edu.mayo.bmi.uima.core.type.syntax.BaseToken;
+
+public class OpenNLPPOSCollectionReaderTests {
+
+ @Test
+ public void testReader() throws ResourceInitializationException, IOException, CollectionException {
+ CollectionReader collectionReader = TestUtil.getCR(new File("test/desc/OpenNLPPOSCollectionReader.xml"));
+ AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/NullAnnotator.xml"));
+ JCas jCas = analysisEngine.newJCas();
+ collectionReader.getNext(jCas.getCas());
+
+ BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals(0, baseToken.getBegin());
+ assertEquals(1, baseToken.getEnd());
+ assertEquals("A", baseToken.getCoveredText());
+ assertEquals("A", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals("farmer", baseToken.getCoveredText());
+ assertEquals("B", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("went", baseToken.getCoveredText());
+ assertEquals("CC", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("trotting", baseToken.getCoveredText());
+ assertEquals("DDD", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
+ assertEquals("upon_A", baseToken.getCoveredText());
+ assertEquals("E", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
+ assertEquals("his", baseToken.getCoveredText());
+ assertEquals("EE", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
+ assertEquals(".", baseToken.getCoveredText());
+ assertEquals(".", baseToken.getPartOfSpeech());
+
+ CollectionException ce = null;
+ jCas = analysisEngine.newJCas();
+ try {
+ collectionReader.getNext(jCas.getCas());
+ } catch(CollectionException e) {
+ ce = e;
+ }
+ assertNotNull(ce);
+
+ jCas = analysisEngine.newJCas();
+ collectionReader.getNext(jCas.getCas());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals("A_", baseToken.getCoveredText());
+ assertEquals("A", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals("_", baseToken.getCoveredText());
+ assertEquals("B", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("_B_", baseToken.getCoveredText());
+ assertEquals("C", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("B", baseToken.getCoveredText());
+ assertEquals("_", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
+ assertEquals("__", baseToken.getCoveredText());
+ assertEquals("_", baseToken.getPartOfSpeech());
+ }
+
+ @Test
+ public void testLoadWordsOnly() throws ResourceInitializationException, IOException, CollectionException {
+ CollectionReader collectionReader = TestUtil.getCR(new File("test/desc/OpenNLPPOSCollectionReader2.xml"));
+ AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/NullAnnotator.xml"));
+ JCas jCas = analysisEngine.newJCas();
+ collectionReader.getNext(jCas.getCas());
+
+ BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals(0, baseToken.getBegin());
+ assertEquals(1, baseToken.getEnd());
+ assertEquals("A", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals("farmer", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("went", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("trotting", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
+ assertEquals("upon_A", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
+ assertEquals("his", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
+ assertEquals(".", baseToken.getCoveredText());
+ assertNull(baseToken.getPartOfSpeech());
+
+ CollectionException ce = null;
+ jCas = analysisEngine.newJCas();
+ try {
+ collectionReader.getNext(jCas.getCas());
+ } catch(CollectionException e) {
+ ce = e;
+ }
+ assertNotNull(ce);
+
+ }
+
+}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/POSTaggerTests.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/POS%20tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/POSTaggerTests.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/POSTaggerTests.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/POSTaggerTests.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2009 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -21,206 +14,206 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.mayo.bmi.uima.pos_tagger.test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import java.io.File;
-import java.util.Iterator;
-
-import org.apache.uima.analysis_engine.AnalysisEngine;
-import org.apache.uima.jcas.JCas;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.junit.Test;
-
-import edu.mayo.bmi.uima.core.test.TestUtil;
-import edu.mayo.bmi.uima.core.type.syntax.BaseToken;
-
-public class POSTaggerTests {
-
- @Test
- public void testTagger() throws ResourceInitializationException {
-
- //this tests the tagging model with no tag dictionary
- AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate.xml"));
- JCas jCas = TestUtil.processAE(analysisEngine, "A farmer went trotting upon his gray mare, Bumpety, bumpety, bump, With his daughter behind him, so rosy and fair, Lumpety, lumpety, lump.");
-
- BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals(0, baseToken.getBegin());
- assertEquals(1, baseToken.getEnd());
- assertEquals("A", baseToken.getCoveredText());
- assertEquals("DT", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals(2, baseToken.getBegin());
- assertEquals(8, baseToken.getEnd());
- assertEquals("farmer", baseToken.getCoveredText());
- assertEquals("JJ", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("went", baseToken.getCoveredText());
- assertEquals("JJ", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("trotting", baseToken.getCoveredText());
- assertEquals("NN", baseToken.getPartOfSpeech());
-
- Iterator<?> baseTokenItr = jCas.getJFSIndexRepository().getAnnotationIndex(BaseToken.type).iterator();
- while(baseTokenItr.hasNext()) {
- baseToken = (BaseToken) baseTokenItr.next();
- assertNotNull(baseToken.getPartOfSpeech());
- }
- }
-
- /*
- * For aiding debug
- */
- private void printPosTags(JCas jCas) {
- BaseToken baseToken;
- // Print the pos tag assigned to each token
- for (int i=0; i < TestUtil.getFeatureStructureSize(jCas, BaseToken.class) ; i++) {
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, i);
- System.out.println(baseToken.getCoveredText() + " part of speech = " + baseToken.getPartOfSpeech());
- }
- }
- /**
- * The first test uses a tag dictionary as case sensitive. So, all the words should be constrained to have the tag "IN"<br>
- * The second test uses the same tag dictionary as case sensitive. However, all the words in the input string have had their
- * case modified so that the tags should not be constrained to be "IN" - but instead simply be whatever the model chooses
- * for them.<br>
- * The third test uses the same tag dictionary as case *insensitive* with the same input string used in the second test. Because
- * the dictionary is case insensitive the tags should be constrained to be "IN".
- * @throws ResourceInitializationException
- */
- @Test
- public void testTagDictionary() throws ResourceInitializationException {
-
- AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate2.xml"));
- JCas jCas = TestUtil.processAE(analysisEngine, "Use of new biologic markers in the ovulation induction.");
-
- //TEST1
- BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals("Use", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals("of", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("new", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("biologic", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
- assertEquals("markers", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
- assertEquals("in", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
- assertEquals("the", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 7);
- assertEquals("ovulation", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 8);
- assertEquals("induction", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 9);
- assertEquals(".", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- Iterator<?> baseTokenItr = jCas.getJFSIndexRepository().getAnnotationIndex(BaseToken.type).iterator();
- while(baseTokenItr.hasNext()) {
- baseToken = (BaseToken) baseTokenItr.next();
- assertNotNull(baseToken.getPartOfSpeech());
- }
-
- //TEST2
- analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate2.xml"));
- jCas = TestUtil.processAE(analysisEngine, "use Of neW Biologic Markers IN The oVULation inductiOn.");
-
- // printPosTags(jCas); // output all the tags so if one of the early ones fail, you still get to see the others
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals("use", baseToken.getCoveredText());
- assertEquals("NN", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals("Of", baseToken.getCoveredText());
- assertEquals("CD", baseToken.getPartOfSpeech()); // NN
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("neW", baseToken.getCoveredText());
- assertEquals("JJ", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("Biologic", baseToken.getCoveredText());
- assertEquals("NN", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
- assertEquals("Markers", baseToken.getCoveredText());
- assertEquals("NNS", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
- assertEquals("IN", baseToken.getCoveredText());
- assertEquals("VBP", baseToken.getPartOfSpeech()); // IN
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
- assertEquals("The", baseToken.getCoveredText());
- assertEquals("DT", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 7);
- assertEquals("oVULation", baseToken.getCoveredText());
- assertEquals("NN", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 8);
- assertEquals("inductiOn", baseToken.getCoveredText());
- assertEquals("NN", baseToken.getPartOfSpeech());
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 9);
- assertEquals(".", baseToken.getCoveredText());
- assertEquals("IN", baseToken.getPartOfSpeech());
-
-
-
- //TEST3
- analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate3.xml"));
- jCas = TestUtil.processAE(analysisEngine, "use Of neW Biologic Markers IN The oVULation inductiOn.");
-
- // This assertion should really expect "IN" - but the POSDictionary class does not read in the
- // dictionary in a case insensitive way. The word "Use" is the only word in the tag dictionary
- // that is not all lower case and so "use" effectively doesn't exist in the dictionary.
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
- assertEquals("NN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 7);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 8);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 9);
- assertEquals("IN", baseToken.getPartOfSpeech());
-
- }
-
-}
+package edu.mayo.bmi.uima.pos_tagger.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.util.Iterator;
+
+import org.apache.uima.analysis_engine.AnalysisEngine;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.junit.Test;
+
+import edu.mayo.bmi.uima.core.test.TestUtil;
+import edu.mayo.bmi.uima.core.type.syntax.BaseToken;
+
+public class POSTaggerTests {
+
+ @Test
+ public void testTagger() throws ResourceInitializationException {
+
+ //this tests the tagging model with no tag dictionary
+ AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate.xml"));
+ JCas jCas = TestUtil.processAE(analysisEngine, "A farmer went trotting upon his gray mare, Bumpety, bumpety, bump, With his daughter behind him, so rosy and fair, Lumpety, lumpety, lump.");
+
+ BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals(0, baseToken.getBegin());
+ assertEquals(1, baseToken.getEnd());
+ assertEquals("A", baseToken.getCoveredText());
+ assertEquals("DT", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals(2, baseToken.getBegin());
+ assertEquals(8, baseToken.getEnd());
+ assertEquals("farmer", baseToken.getCoveredText());
+ assertEquals("JJ", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("went", baseToken.getCoveredText());
+ assertEquals("JJ", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("trotting", baseToken.getCoveredText());
+ assertEquals("NN", baseToken.getPartOfSpeech());
+
+ Iterator<?> baseTokenItr = jCas.getJFSIndexRepository().getAnnotationIndex(BaseToken.type).iterator();
+ while(baseTokenItr.hasNext()) {
+ baseToken = (BaseToken) baseTokenItr.next();
+ assertNotNull(baseToken.getPartOfSpeech());
+ }
+ }
+
+ /*
+ * For aiding debug
+ */
+ private void printPosTags(JCas jCas) {
+ BaseToken baseToken;
+ // Print the pos tag assigned to each token
+ for (int i=0; i < TestUtil.getFeatureStructureSize(jCas, BaseToken.class) ; i++) {
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, i);
+ System.out.println(baseToken.getCoveredText() + " part of speech = " + baseToken.getPartOfSpeech());
+ }
+ }
+ /**
+ * The first test uses a tag dictionary as case sensitive. So, all the words should be constrained to have the tag "IN"<br>
+ * The second test uses the same tag dictionary as case sensitive. However, all the words in the input string have had their
+ * case modified so that the tags should not be constrained to be "IN" - but instead simply be whatever the model chooses
+ * for them.<br>
+ * The third test uses the same tag dictionary as case *insensitive* with the same input string used in the second test. Because
+ * the dictionary is case insensitive the tags should be constrained to be "IN".
+ * @throws ResourceInitializationException
+ */
+ @Test
+ public void testTagDictionary() throws ResourceInitializationException {
+
+ AnalysisEngine analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate2.xml"));
+ JCas jCas = TestUtil.processAE(analysisEngine, "Use of new biologic markers in the ovulation induction.");
+
+ //TEST1
+ BaseToken baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals("Use", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals("of", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("new", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("biologic", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
+ assertEquals("markers", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
+ assertEquals("in", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
+ assertEquals("the", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 7);
+ assertEquals("ovulation", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 8);
+ assertEquals("induction", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 9);
+ assertEquals(".", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ Iterator<?> baseTokenItr = jCas.getJFSIndexRepository().getAnnotationIndex(BaseToken.type).iterator();
+ while(baseTokenItr.hasNext()) {
+ baseToken = (BaseToken) baseTokenItr.next();
+ assertNotNull(baseToken.getPartOfSpeech());
+ }
+
+ //TEST2
+ analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate2.xml"));
+ jCas = TestUtil.processAE(analysisEngine, "use Of neW Biologic Markers IN The oVULation inductiOn.");
+
+ // printPosTags(jCas); // output all the tags so if one of the early ones fail, you still get to see the others
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals("use", baseToken.getCoveredText());
+ assertEquals("NN", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals("Of", baseToken.getCoveredText());
+ assertEquals("CD", baseToken.getPartOfSpeech()); // NN
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("neW", baseToken.getCoveredText());
+ assertEquals("JJ", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("Biologic", baseToken.getCoveredText());
+ assertEquals("NN", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
+ assertEquals("Markers", baseToken.getCoveredText());
+ assertEquals("NNS", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
+ assertEquals("IN", baseToken.getCoveredText());
+ assertEquals("VBP", baseToken.getPartOfSpeech()); // IN
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
+ assertEquals("The", baseToken.getCoveredText());
+ assertEquals("DT", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 7);
+ assertEquals("oVULation", baseToken.getCoveredText());
+ assertEquals("NN", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 8);
+ assertEquals("inductiOn", baseToken.getCoveredText());
+ assertEquals("NN", baseToken.getPartOfSpeech());
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 9);
+ assertEquals(".", baseToken.getCoveredText());
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+
+
+ //TEST3
+ analysisEngine = TestUtil.getAE(new File("test/desc/POSTaggerAggregate3.xml"));
+ jCas = TestUtil.processAE(analysisEngine, "use Of neW Biologic Markers IN The oVULation inductiOn.");
+
+ // This assertion should really expect "IN" - but the POSDictionary class does not read in the
+ // dictionary in a case insensitive way. The word "Use" is the only word in the tag dictionary
+ // that is not all lower case and so "use" effectively doesn't exist in the dictionary.
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 0);
+ assertEquals("NN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 1);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 2);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 3);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 4);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 5);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 6);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 7);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 8);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ baseToken = TestUtil.getFeatureStructureAtIndex(jCas, BaseToken.class, 9);
+ assertEquals("IN", baseToken.getPartOfSpeech());
+
+ }
+
+}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/TagDictionaryCreatorTests.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/POS%20tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/TagDictionaryCreatorTests.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/TagDictionaryCreatorTests.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/POS tagger/test/src/edu/mayo/bmi/uima/pos_tagger/test/TagDictionaryCreatorTests.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2009 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -21,87 +14,87 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package edu.mayo.bmi.uima.pos_tagger.test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Set;
-
-import org.junit.Test;
-
-import edu.mayo.bmi.uima.pos_tagger.TagDictionaryCreator;
-
-public class TagDictionaryCreatorTests {
-
- @Test
- public void testCreateTagDictionary() throws FileNotFoundException, IOException {
- BufferedReader input = new BufferedReader(new FileReader("test/data/unit-test-2lines-training-data.txt"));
- HashMap<String, Set<String>> tagDictionaryData = TagDictionaryCreator.createTagDictionary(input, true);
-
- Set<String> tags = tagDictionaryData.get("IL-2");
- assertEquals(3, tags.size());
- boolean nn = false;
- boolean cc = false;
- boolean in = false;
- for(String tag : tags) {
- if(tag.equals("NN"))
- nn = true;
- if(tag.equals("CC"))
- cc = true;
- if(tag.equals("IN"))
- in = true;
- }
- assertTrue(nn);
- assertTrue(cc);
- assertTrue(in);
-
- input = new BufferedReader(new FileReader("test/data/unit-test-2lines-training-data.txt"));
- tagDictionaryData = TagDictionaryCreator.createTagDictionary(input, false);
-
- tags = tagDictionaryData.get("il-2");
- assertEquals(3, tags.size());
- nn = false;
- cc = false;
- in = false;
- for(String tag : tags) {
- if(tag.equals("NN"))
- nn = true;
- if(tag.equals("CC"))
- cc = true;
- if(tag.equals("IN"))
- in = true;
- }
- assertTrue(nn);
- assertTrue(cc);
- assertTrue(in);
-
- tags = tagDictionaryData.get("surface");
- assertEquals("NN", tags.iterator().next());
-
- }
-
-
- @Test
- public void testMain() throws FileNotFoundException, IOException{
- String[] args = new String[] { "test/data/unit-test-2lines-training-data.txt", "test/data/output/unit-test-2lines-tag-dictionary.txt", "true"};
- TagDictionaryCreator.main(args);
- BufferedReader input = new BufferedReader(new FileReader("test/data/output/unit-test-2lines-tag-dictionary.txt"));
- String line;
- int lines = 0;
- while((line = input.readLine())!= null) {
- lines++;
- if(line.startsWith("IL-2 "))
- assertEquals("IL-2 CC IN NN", line);
- if(line.startsWith("requires "))
- assertEquals("requires VBZ", line);
- }
- assertEquals(36, lines);
- }
-
-}
+package edu.mayo.bmi.uima.pos_tagger.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Set;
+
+import org.junit.Test;
+
+import edu.mayo.bmi.uima.pos_tagger.TagDictionaryCreator;
+
+public class TagDictionaryCreatorTests {
+
+ @Test
+ public void testCreateTagDictionary() throws FileNotFoundException, IOException {
+ BufferedReader input = new BufferedReader(new FileReader("test/data/unit-test-2lines-training-data.txt"));
+ HashMap<String, Set<String>> tagDictionaryData = TagDictionaryCreator.createTagDictionary(input, true);
+
+ Set<String> tags = tagDictionaryData.get("IL-2");
+ assertEquals(3, tags.size());
+ boolean nn = false;
+ boolean cc = false;
+ boolean in = false;
+ for(String tag : tags) {
+ if(tag.equals("NN"))
+ nn = true;
+ if(tag.equals("CC"))
+ cc = true;
+ if(tag.equals("IN"))
+ in = true;
+ }
+ assertTrue(nn);
+ assertTrue(cc);
+ assertTrue(in);
+
+ input = new BufferedReader(new FileReader("test/data/unit-test-2lines-training-data.txt"));
+ tagDictionaryData = TagDictionaryCreator.createTagDictionary(input, false);
+
+ tags = tagDictionaryData.get("il-2");
+ assertEquals(3, tags.size());
+ nn = false;
+ cc = false;
+ in = false;
+ for(String tag : tags) {
+ if(tag.equals("NN"))
+ nn = true;
+ if(tag.equals("CC"))
+ cc = true;
+ if(tag.equals("IN"))
+ in = true;
+ }
+ assertTrue(nn);
+ assertTrue(cc);
+ assertTrue(in);
+
+ tags = tagDictionaryData.get("surface");
+ assertEquals("NN", tags.iterator().next());
+
+ }
+
+
+ @Test
+ public void testMain() throws FileNotFoundException, IOException{
+ String[] args = new String[] { "test/data/unit-test-2lines-training-data.txt", "test/data/output/unit-test-2lines-tag-dictionary.txt", "true"};
+ TagDictionaryCreator.main(args);
+ BufferedReader input = new BufferedReader(new FileReader("test/data/output/unit-test-2lines-tag-dictionary.txt"));
+ String line;
+ int lines = 0;
+ while((line = input.readLine())!= null) {
+ lines++;
+ if(line.startsWith("IL-2 "))
+ assertEquals("IL-2 CC IN NN", line);
+ if(line.startsWith("requires "))
+ assertEquals("requires VBZ", line);
+ }
+ assertEquals(36, lines);
+ }
+
+}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/DependencyNodeWriter.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/DependencyNodeWriter.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/DependencyNodeWriter.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/DependencyNodeWriter.java Wed Oct 31 05:26:43 2012
@@ -1,19 +1,12 @@
package edu.mayo.bmi.nlp.parser;
/*
- * Copyright: (c) 2010 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -22,8 +15,8 @@ package edu.mayo.bmi.nlp.parser;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-
+
+
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
@@ -40,63 +33,63 @@ import org.apache.uima.resource.Resource
import edu.mayo.bmi.uima.core.type.syntax.ConllDependencyNode;
import edu.mayo.bmi.uima.core.type.textspan.Sentence;
import edu.mayo.bmi.uima.core.util.DocumentIDAnnotationUtil;
-
-
-/**
- * CasConsumer that writes a JCas (the current view) to an xml file
- *
- * @author Mayo Clinic
- */
-public class DependencyNodeWriter extends CasConsumer_ImplBase
-{
- // LOG4J logger based on class name
- private Logger iv_logger = Logger.getLogger(getClass().getName());
-
+
+
+/**
+ * CasConsumer that writes a JCas (the current view) to an xml file
+ *
+ * @author Mayo Clinic
+ */
+public class DependencyNodeWriter extends CasConsumer_ImplBase
+{
+ // LOG4J logger based on class name
+ private Logger iv_logger = Logger.getLogger(getClass().getName());
+
private String iv_outputDir = null;
- private String iv_outputFormat = null;
-
- // iv_procCount is used to name the output files sequentially if there
- // is a problem with naming based on source names
- private int iv_procCount = 0;
-
-
- /**
- * Read in configuration parameters
- */
- public void initialize() throws ResourceInitializationException {
+ private String iv_outputFormat = null;
+
+ // iv_procCount is used to name the output files sequentially if there
+ // is a problem with naming based on source names
+ private int iv_procCount = 0;
+
+
+ /**
+ * Read in configuration parameters
+ */
+ public void initialize() throws ResourceInitializationException {
iv_outputDir = (String) getConfigParameterValue("outputDir");
- iv_outputFormat = (String) getConfigParameterValue("outputFormat");
- }
-
-
- /**
- * Write a tab-delimited file containing data from the view.
- * The file name will come from the DocumentID annotation,
- * which is associated with a view.
- */
- private void processView(JCas jCas) throws Exception {
- // String docText = view.getDocumentText();
-
- String docName = DocumentIDAnnotationUtil.getDocumentID(jCas);
-
- File outputFile;
- if (docName==null) {
- docName = "doc" + iv_procCount + "." + iv_outputFormat.toLowerCase();
- }
- else {
- docName = docName + "." + iv_outputFormat.toLowerCase();
- // if (!docName.endsWith(".xml")) {
- // docName = docName + ".xml";
- // }
- }
-
-// OutputStream out=null;
- try {
- File outputDir = new File(iv_outputDir);
- outputDir.mkdirs();
- outputFile = new File(iv_outputDir + File.separatorChar + docName);
-// out = new FileOutputStream(outputFile);
-// XCASSerializer.serialize(view.getCas(), out, true); // true -> formats the output
+ iv_outputFormat = (String) getConfigParameterValue("outputFormat");
+ }
+
+
+ /**
+ * Write a tab-delimited file containing data from the view.
+ * The file name will come from the DocumentID annotation,
+ * which is associated with a view.
+ */
+ private void processView(JCas jCas) throws Exception {
+ // String docText = view.getDocumentText();
+
+ String docName = DocumentIDAnnotationUtil.getDocumentID(jCas);
+
+ File outputFile;
+ if (docName==null) {
+ docName = "doc" + iv_procCount + "." + iv_outputFormat.toLowerCase();
+ }
+ else {
+ docName = docName + "." + iv_outputFormat.toLowerCase();
+ // if (!docName.endsWith(".xml")) {
+ // docName = docName + ".xml";
+ // }
+ }
+
+// OutputStream out=null;
+ try {
+ File outputDir = new File(iv_outputDir);
+ outputDir.mkdirs();
+ outputFile = new File(iv_outputDir + File.separatorChar + docName);
+// out = new FileOutputStream(outputFile);
+// XCASSerializer.serialize(view.getCas(), out, true); // true -> formats the output
outputFile.createNewFile();
BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile));
@@ -160,33 +153,33 @@ public class DependencyNodeWriter extend
}
bw.flush();
- }
- finally {
- iv_procCount++;
-// if (out != null) {
-// out.close();
-// }
- }
-
- }
-
-
- /**
- * Create an xml file from the data in the cas.
- */
- public void processCas(CAS cas) throws ResourceProcessException {
-
- iv_logger.info("Started");
-
- try {
-
- JCas currentView = cas.getCurrentView().getJCas();
- processView(currentView);
-
- } catch (Exception e) {
- throw new ResourceProcessException(e);
- }
-
- }
-
-}
\ No newline at end of file
+ }
+ finally {
+ iv_procCount++;
+// if (out != null) {
+// out.close();
+// }
+ }
+
+ }
+
+
+ /**
+ * Create an xml file from the data in the cas.
+ */
+ public void processCas(CAS cas) throws ResourceProcessException {
+
+ iv_logger.info("Started");
+
+ try {
+
+ JCas currentView = cas.getCurrentView().getJCas();
+ processView(currentView);
+
+ } catch (Exception e) {
+ throw new ResourceProcessException(e);
+ }
+
+ }
+
+}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserDependencyParserAE.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserDependencyParserAE.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserDependencyParserAE.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserDependencyParserAE.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2012 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -78,7 +71,7 @@ public class ClearParserDependencyParser
// Default model values
public static final String DEFAULT_MODEL_FILE_NAME = "/resources/dependency/conll-2009-dev-shift-pop.jar";
-
+
public static final String ENG_LEMMATIZER_DATA_FILE = "/resources/lemmatizer/wordnet-3.0-lemma-data.jar";
public static final String DEFAULT_PARSER_ALGORITHM_NAME = AbstractDepParser.ALG_SHIFT_POP;
@@ -165,7 +158,7 @@ public class ClearParserDependencyParser
ArrayList<ConllDependencyNode> nodes = ClearDependencyUtility.convert(jCas, tree, sentence, tokens);
ClearDependencyUtility.addToIndexes(jCas, nodes);
}
-
+
}
}
Modified: incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserSemanticRoleLabelerAE.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserSemanticRoleLabelerAE.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserSemanticRoleLabelerAE.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserSemanticRoleLabelerAE.java Wed Oct 31 05:26:43 2012
@@ -1,18 +1,11 @@
/*
- * Copyright: (c) 2012 Mayo Foundation for Medical Education and
- * Research (MFMER). All rights reserved. MAYO, MAYO CLINIC, and the
- * triple-shield Mayo logo are trademarks and service marks of MFMER.
- *
- * Except as contained in the copyright notice above, or as used to identify
- * MFMER as the author of this software, the trade names, trademarks, service
- * marks, or product names of the copyright holder shall not be used in
- * advertising, promotion or otherwise in connection with this software without
- * prior written authorization of the copyright holder.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -113,14 +106,14 @@ public class ClearParserSemanticRoleLabe
@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
super.initialize(context);
-
+
logger.info("using Morphy analysis? " + useLemmatizer);
try {
if (useLemmatizer) {
// Note: If lemmatizer data file is not specified, then use lemmas from the BaseToken normalizedToken field.
// Initialize lemmatizer
- URL lemmatizerDataFileURL = this.getClass().getResource(ENG_LEMMATIZER_DATA_FILE);
+ URL lemmatizerDataFileURL = this.getClass().getResource(ENG_LEMMATIZER_DATA_FILE);
lemmatizer = new MorphEnAnalyzer(new URL(lemmatizerDataFileURL.toString()));
}
@@ -150,7 +143,7 @@ public class ClearParserSemanticRoleLabe
for (ConllDependencyNode depNode : JCasUtil.selectCovered(jCas, ConllDependencyNode.class, sentence)) {
//if (depNode instanceof TopDependencyNode) {
if (depNode.getHead() == null) {
- // A node without the head is the head of the sentence
+ // A node without the head is the head of the sentence
depNodeToID.put(depNode, 0);
} else {
depNodeToID.put(depNode, nodeId);
Modified: incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserUtil.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserUtil.java?rev=1403989&r1=1403988&r2=1403989&view=diff
==============================================================================
--- incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserUtil.java (original)
+++ incubator/ctakes/branches/SHARPn-cTAKES/clearparser-wrapper/src/edu/mayo/bmi/nlp/parser/ae/ClearParserUtil.java Wed Oct 31 05:26:43 2012
@@ -1,26 +1,11 @@
/*
- * Copyright (c) 2011, Regents of the University of Colorado
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
- * Neither the name of the University of Colorado at Boulder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
package edu.mayo.bmi.nlp.parser.ae;