You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by mb...@apache.org on 2008/01/08 14:40:49 UTC
svn commit: r609977 -
/incubator/uima/sandbox/trunk/DictionaryAnnotator/src/main/java/org/apache/uima/annotator/dict_annot/dictionary/impl/DictionaryCreator.java
Author: mbaessler
Date: Tue Jan 8 05:40:48 2008
New Revision: 609977
URL: http://svn.apache.org/viewvc?rev=609977&view=rev
Log:
UIMA-691
add dictionary creation using a command line program
https://issues.apache.org/jira/browse/UIMA-691
Added:
incubator/uima/sandbox/trunk/DictionaryAnnotator/src/main/java/org/apache/uima/annotator/dict_annot/dictionary/impl/DictionaryCreator.java
Added: incubator/uima/sandbox/trunk/DictionaryAnnotator/src/main/java/org/apache/uima/annotator/dict_annot/dictionary/impl/DictionaryCreator.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/DictionaryAnnotator/src/main/java/org/apache/uima/annotator/dict_annot/dictionary/impl/DictionaryCreator.java?rev=609977&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/DictionaryAnnotator/src/main/java/org/apache/uima/annotator/dict_annot/dictionary/impl/DictionaryCreator.java (added)
+++ incubator/uima/sandbox/trunk/DictionaryAnnotator/src/main/java/org/apache/uima/annotator/dict_annot/dictionary/impl/DictionaryCreator.java Tue Jan 8 05:40:48 2008
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.annotator.dict_annot.dictionary.impl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.StringTokenizer;
+
+import org.apache.uima.UIMAFramework;
+import org.apache.uima.analysis_engine.AnalysisEngine;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.internal.util.CommandLineParser;
+import org.apache.uima.pear.tools.PackageBrowser;
+import org.apache.uima.pear.tools.PackageInstaller;
+import org.apache.uima.resource.ResourceSpecifier;
+import org.apache.uima.util.XMLInputSource;
+
+/**
+ * The DicrionaryCreator creates an dictionary XML file based on the given input
+ * parameters. The input file must contain one dictionary entry for each line.
+ * The line is tokenized into multi words either with the specified tokenizer
+ * PEAR file or with the separator character.
+ *
+ * The dictionary output XML file (UTF-8) must be updated after the creation
+ * with some metadata information.
+ */
+public class DictionaryCreator {
+
+ // input file command line parameter
+ private static final String INPUT_FILE_PARAM = "-input";
+
+ // output file command line parameter
+ private static final String OUTPUT_FILE_PARAM = "-output";
+
+ // input file encoding command line parameter
+ private static final String INPUT_FILE_ENCODING_PARAM = "-encoding";
+
+ // input file language command line parameter
+ private static final String INPUT_FILE_LANGUAGE_PARAM = "-lang";
+
+ // tokenizer command line parameter
+ private static final String TOKENIZER_PARAM = "-tokenizer";
+
+ // tokenizer token type command line parameter
+ private static final String TOKEN_TYPE_PARAM = "-tokenType";
+
+ // token separator character command line parameter
+ private static final String SEPARATOR_CHAR_PARAM = "-separator";
+
+ // default separator character
+ private static final String SEPARATOR_CHAR = " ";
+
+ /**
+ * creates the CommandLine parser used to parse the DictionaryConverter
+ * command line.
+ *
+ * @return returns the CommandLineParser for the DictionaryConverter
+ */
+ private static final CommandLineParser createCmdLineParser() {
+ CommandLineParser parser = new CommandLineParser();
+ parser.addParameter(INPUT_FILE_PARAM, true);
+ parser.addParameter(INPUT_FILE_LANGUAGE_PARAM, true);
+ parser.addParameter(OUTPUT_FILE_PARAM, true);
+ parser.addParameter(INPUT_FILE_ENCODING_PARAM, true);
+ parser.addParameter(TOKENIZER_PARAM, true);
+ parser.addParameter(TOKEN_TYPE_PARAM, true);
+ parser.addParameter(SEPARATOR_CHAR_PARAM, true);
+ return parser;
+ }
+
+ /**
+ * prints the DictionaryConverter usage to the command line
+ */
+ private static final void printUsage() {
+ System.out
+ .println("Usage: java org.apache.uima.annotator.dict_annot.dictionary.impl.DictionaryCreator -input <InputFile> -encoding <InputFileEncoding> -output <OutputFile> [-tokenizer <TokenizerPear> -tokenType <tokenType>] [-separator <separatorChar>] ");
+ System.out.println("Additional optional parameters:");
+ System.out.println(" -lang <dictionaryLanguage>");
+
+ }
+
+ /**
+ * checks the command line syntax for errors
+ *
+ * @param clp
+ * current CommandLineParser
+ *
+ * @return returns false in case of command line syntax errors, otherwise
+ * true
+ */
+ private static final boolean checkCmdLineSyntax(CommandLineParser clp) {
+ boolean error = false;
+ // check mandatory command line parameters
+ if (!clp.isInArgsList(INPUT_FILE_PARAM)) {
+ System.err.println("InputFile parameter -input is missing");
+ error = true;
+ }
+ if (!clp.isInArgsList(INPUT_FILE_ENCODING_PARAM)) {
+ System.err
+ .println("InputFile encoding parameter -encoding is missing");
+ error = true;
+ }
+ if (!clp.isInArgsList(OUTPUT_FILE_PARAM)) {
+ System.err.println("OutputFile parameter -output is missing");
+ error = true;
+ }
+ if (clp.isInArgsList(TOKENIZER_PARAM)) {
+ if (!clp.isInArgsList(TOKEN_TYPE_PARAM)) {
+ System.err
+ .println("If a tokenizer is used, the -tokenType paramter must be specified");
+ error = true;
+ }
+ }
+
+ // if there was an error, return false
+ if (error == true) {
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * main method for DictionaryCreator
+ *
+ * @param args
+ * command line arguments
+ */
+ public static void main(String[] args) {
+
+ // create command line parser
+ CommandLineParser clp = createCmdLineParser();
+ try {
+ // parse command line
+ clp.parseCmdLine(args);
+ // check command line syntax
+ if (!checkCmdLineSyntax(clp)) {
+ printUsage();
+ System.exit(-1);
+ }
+ } catch (Exception e) {
+ System.err.println("Error parsing command line: " + e.getMessage());
+ }
+
+ // get command line arguments
+ String inputFile = clp.getParamArgument(INPUT_FILE_PARAM);
+ String language = clp.getParamArgument(INPUT_FILE_LANGUAGE_PARAM);
+ String encoding = clp.getParamArgument(INPUT_FILE_ENCODING_PARAM);
+ String outputFile = clp.getParamArgument(OUTPUT_FILE_PARAM);
+ String tokenizerFile = clp.getParamArgument(TOKENIZER_PARAM);
+ String tokenTypeStr = clp.getParamArgument(TOKEN_TYPE_PARAM);
+ String separatorChar = clp.getParamArgument(SEPARATOR_CHAR_PARAM);
+
+ // check input file command line argument
+ File inFile = new File(inputFile);
+ if (!inFile.canRead()) {
+ System.err.println("Error: Input file " + inputFile
+ + " cannot be read!");
+ System.exit(-1);
+ }
+
+ // check tokenizer pear command line arguments
+ AnalysisEngine ae = null;
+ Type tokenType = null;
+ CAS cas = null;
+ if (tokenizerFile != null) {
+ // if a tokenizer is specified, check if the file can be read
+ File pearFile = new File(tokenizerFile);
+ if (!pearFile.canRead()) {
+ System.err.println("Error: Tokenizer file " + tokenizerFile
+ + " cannot be read!");
+ System.exit(-1);
+ }
+ if (tokenTypeStr == null) {
+ System.err.println("Error: Tokenizer tokenType not specified");
+ System.exit(-1);
+ }
+ try {
+ // create temp directory to install PEAR
+ File tempDir = new File(".", "~tokenizer_temp_install");
+ tempDir.deleteOnExit();
+ tempDir.mkdir();
+
+ // Install PEAR package
+ PackageBrowser instPear = PackageInstaller.installPackage(tempDir,
+ pearFile, true);
+
+ // Create analysis engine from the installed PEAR package
+ XMLInputSource in = new XMLInputSource(instPear
+ .getComponentPearDescPath());
+ ResourceSpecifier specifier = UIMAFramework.getXMLParser()
+ .parseResourceSpecifier(in);
+ ae = UIMAFramework.produceAnalysisEngine(specifier);
+
+ // create CAS and initialize tokenType
+ cas = ae.newCAS();
+ tokenType = cas.getTypeSystem().getType(tokenTypeStr);
+
+ } catch (Exception ex) {
+ System.err.println("Error creating tokenizer: " + ex.getMessage());
+ ex.printStackTrace();
+ System.exit(-1);
+ }
+ }
+
+ // check separator char command line argument
+ if (separatorChar == null && ae == null) {
+ // use default separator character
+ separatorChar = SEPARATOR_CHAR;
+ }
+
+ try {
+ // initialize input and output files
+ BufferedReader reader = new BufferedReader(new InputStreamReader(
+ new FileInputStream(inputFile), encoding));
+ BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(outputFile), "UTF-8"));
+
+ // write dictionary XML lead in
+ writer
+ .write("<dictionary xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation=\"dictionary.xsd\">\n");
+ writer.write("<typeCollection>\n");
+ writer
+ .write("<dictionaryMetaData caseNormalization=\"true\" multiWordEntries=\"true\" multiWordSeparator=\"\t\"/>");
+ if (language != null) {
+ writer.write("<languageId>" + language + "</languageId>\n");
+ }
+ writer.write("<typeDescription>\n");
+ writer
+ .write("<typeName> ADD DICTIONARY OUTPUT TYPE HERE</typeName>\n");
+ writer.write("</typeDescription>\n");
+ writer.write("<entries>\n");
+
+ // start adding dictionary entries
+ String line = reader.readLine();
+ while (line != null) {
+
+ // multi workd string buffer - contains the tokens for each entry
+ StringBuffer multiWordString = new StringBuffer();
+
+ // tokenize entry line
+ if (ae != null) { // use tokenizer
+ cas.setDocumentText(line);
+ if (language != null) {
+ cas.setDocumentLanguage(language);
+ }
+ // tokenize line
+ ae.process(cas);
+
+ // read results
+ FSIterator it = cas.getAnnotationIndex(tokenType).iterator();
+ while (it.hasNext()) {
+ multiWordString.append(((AnnotationFS) it.next())
+ .getCoveredText());
+ multiWordString.append("\t");
+ }
+ cas.reset();
+
+ } else { // use separator char
+
+ StringTokenizer tokenizer = new StringTokenizer(line,
+ separatorChar);
+ while (tokenizer.hasMoreTokens()) {
+ multiWordString.append(tokenizer.nextToken());
+ multiWordString.append("\t");
+ }
+ }
+
+ // write dictionary entry to XML
+ writer.write("<entry>\n");
+ writer.write("<key>" + multiWordString.toString().trim()
+ + "</key>\n");
+ writer.write("</entry>\n");
+
+ // get next line
+ line = reader.readLine();
+ } // all dictionary lines are processed
+ reader.close();
+
+ // write dictionary XML lead out
+ writer.write("</entries>\n");
+ writer.write("</typeCollection>\n");
+ writer.write("</dictionary>\n");
+ writer.close();
+
+ System.out.println("The dictionary was sucessfully created at: "
+ + outputFile);
+
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+}