You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by pk...@apache.org on 2013/06/07 22:48:36 UTC
svn commit: r1490815 [2/3] - in /uima/sandbox/ruta/trunk/ruta-ep-textruler:
./ src/main/java/org/apache/uima/ruta/textruler/extension/
src/main/java/org/apache/uima/ruta/textruler/learner/trabal/
Added: uima/sandbox/ruta/trunk/ruta-ep-textruler/src/main/java/org/apache/uima/ruta/textruler/learner/trabal/TrabalLearner.java
URL: http://svn.apache.org/viewvc/uima/sandbox/ruta/trunk/ruta-ep-textruler/src/main/java/org/apache/uima/ruta/textruler/learner/trabal/TrabalLearner.java?rev=1490815&view=auto
==============================================================================
--- uima/sandbox/ruta/trunk/ruta-ep-textruler/src/main/java/org/apache/uima/ruta/textruler/learner/trabal/TrabalLearner.java (added)
+++ uima/sandbox/ruta/trunk/ruta-ep-textruler/src/main/java/org/apache/uima/ruta/textruler/learner/trabal/TrabalLearner.java Fri Jun 7 20:48:36 2013
@@ -0,0 +1,2097 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
+package org.apache.uima.ruta.textruler.learner.trabal;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIterator;
+import org.apache.uima.cas.Feature;
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.cas.text.AnnotationIndex;
+import org.apache.uima.ruta.textruler.core.TextRulerBasicLearner;
+import org.apache.uima.ruta.textruler.core.TextRulerExample;
+import org.apache.uima.ruta.textruler.core.TextRulerExampleDocument;
+import org.apache.uima.ruta.textruler.core.TextRulerExampleDocumentSet;
+import org.apache.uima.ruta.textruler.core.TextRulerStatisticsCollector;
+import org.apache.uima.ruta.textruler.core.TextRulerTarget;
+import org.apache.uima.ruta.textruler.core.TextRulerToolkit;
+import org.apache.uima.ruta.textruler.extension.TextRulerLearnerDelegate;
+import org.apache.uima.ruta.textruler.tools.MemoryWatch;
+
+public class TrabalLearner extends TextRulerBasicLearner {
+
+ public final static String ANNOTATION_TYPE_BASIC = "org.apache.uima.ruta.type.RutaBasic";
+
+ public final static String ANNOTATION_TYPE_ANY = "org.apache.uima.ruta.type.ANY";
+
+ public final static String ANNOTATION_TYPE_FRAME = "org.apache.uima.ruta.type.RutaFrame";
+
+ public final static String PACKAGE = "org.apache.uima";
+
+ public final static String TYPESYSTEM = "de.uniwue.tm.citie.CompleteTypeSystemTypeSystem";
+
+ public final static List<String> FILTERED_FEATURES;
+
+ static {
+ ArrayList<String> tmp = new ArrayList<String>();
+ tmp.add("sofa");
+ tmp.add("begin");
+ tmp.add("end");
+ FILTERED_FEATURES = tmp;
+ }
+
+ public final static String MAX_NUMBER_OF_BASIC_RULES_KEY = "maxNumberOfBasicRules";
+
+ public final static String MAX_NUMBER_OF_RULES_KEY = "maxNumberOfRules";
+
+ public final static String MAX_NUMBER_OF_ITERATIONS_KEY = "maxNumberOfIterations";
+
+ public final static String ALGORITHM_ITERATIONS_KEY = "algorithmIterations";
+
+ public final static String MAX_ERROR_RATE_KEY = "maxErrorRate";
+
+ public final static String ENABLE_FEATURES_KEY = "enableFeatures";
+
+ public final static int MAX_NUMBER_OF_BASIC_RULES = 80;
+
+ public final static int MAX_NUMBER_OF_RULES = 150;
+
+ public final static int MAX_NUMBER_OF_ITERATIONS = 2;
+
+ public final static int ALGORITHM_ITERATIONS = 3;
+
+ public final static double MAX_ERROR_RATE = 0.4;
+
+ public final static boolean ENABLE_FEATURES = false;
+
+ private int maxNumberOfBasicRules = MAX_NUMBER_OF_BASIC_RULES;
+
+ private int maxNumberOfRules = MAX_NUMBER_OF_RULES;
+
+ private int maxNumberOfIterations = MAX_NUMBER_OF_ITERATIONS;
+
+ private int algorithmIterations = ALGORITHM_ITERATIONS;
+
+ private double maxErrorRate = MAX_ERROR_RATE;
+
+ public boolean enableFeatures = ENABLE_FEATURES;
+
+ private String result = ""; // actualResult + tempRules
+
+ private String actualResult = "";
+
+ private List<TrabalRule> bestRulesForStatus = new ArrayList<TrabalRule>();
+
+ private String inputDirectory;
+
+ private String additionalFolderPath;
+
+ private TextRulerExampleDocumentSet additionalDocuments;
+
+ private List<AnnotationError> errors;
+
+ private Map<String, RankedList> positiveExamples;
+
+ private Map<String, Double> idf;
+
+ public TrabalLearner(String inputFolderPath, String additionalFolderPath,
+ String preprocessorTMfile, String tempFolderPath, String[] fullSlotTypeNames,
+ Set<String> filterSet, boolean skip, TextRulerLearnerDelegate delegate) {
+ super(inputFolderPath, preprocessorTMfile, tempFolderPath, fullSlotTypeNames, filterSet,
+ skip, delegate);
+ this.inputDirectory = inputFolderPath;
+ this.additionalFolderPath = additionalFolderPath;
+ }
+
+ /**
+ * Main method. Starts the algorithm.
+ */
+ protected void doRun() {
+ try {
+ getAnalysisEngine();
+ getAdditionalDocuments();
+ if (exampleDocuments.getDocuments().size() != additionalDocuments.getDocuments().size())
+ throw new Exception("Error: Training data doesn't match additional data!");
+ sendStatusUpdateToDelegate("Loading documents...", TextRulerLearnerState.ML_INITIALIZING,
+ true);
+ sendStatusUpdateToDelegate("Comparing documents...", TextRulerLearnerState.ML_RUNNING, true);
+ for (int i = 0; i < algorithmIterations; i++) {
+ actualResult += "BLOCK(Iteration_" + (i + 1) + ") Document{} {\n";
+ sendStatusUpdateToDelegate("Comparing documents...", TextRulerLearnerState.ML_RUNNING, true);
+ idf = createIDF();
+ Map<String, List<AnnotationError>> errorGrps = createErrorGroups();
+ List<TrabalRule> rules = runAlgorithm(errorGrps);
+ if (rules.size() == 0) {
+ actualResult = actualResult.replace("BLOCK(Iteration_" + (i + 1) + ") Document{} {", "");
+ break;
+ }
+ actualResult += "}\n\n";
+ result = actualResult;
+ sendStatusUpdateToDelegate("Finished " + (i + 1) + ". iteration.",
+ TextRulerLearnerState.ML_DONE, true);
+ }
+ Map<String, List<AnnotationError>> errorGrps = createErrorGroups();
+ if (errorGrps.size() > 0) {
+ actualResult += "// Remaining errors \n";
+ for (List<AnnotationError> list : errorGrps.values()) {
+ for (AnnotationError err : list) {
+ actualResult += "\n// " + err;
+ }
+ }
+ }
+ if (new File(tempDirectory).isDirectory())
+ new File(tempDirectory).delete();
+ result = actualResult;
+ sendStatusUpdateToDelegate("Done!", TextRulerLearnerState.ML_DONE, true);
+ } catch (Exception e) {
+ sendStatusUpdateToDelegate(e.getLocalizedMessage(), TextRulerLearnerState.ML_ERROR, true);
+ e.printStackTrace();
+ }
+ }
+
+ private void removeBasics() {
+ for (TextRulerExampleDocument doc : additionalDocuments.getDocuments()) {
+ CAS cas = doc.getCAS();
+ List<AnnotationFS> basics = new ArrayList<AnnotationFS>();
+ for (AnnotationFS a : cas.getAnnotationIndex()) {
+ if (a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ basics.add(a);
+ }
+ }
+ for (AnnotationFS a : basics)
+ cas.removeFsFromIndexes(a);
+ }
+ }
+
+ private void removeBasics(CAS cas) {
+ List<AnnotationFS> basics = new ArrayList<AnnotationFS>();
+ for (AnnotationFS a : cas.getAnnotationIndex()) {
+ if (a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ basics.add(a);
+ }
+ }
+ for (AnnotationFS a : basics)
+ cas.removeFsFromIndexes(a);
+ }
+
+ private Map<String, Double> createIDF() {
+ Map<String, Double> result = new HashMap<String, Double>();
+ List<TextRulerExampleDocument> documents = additionalDocuments.getDocuments();
+ // int counter = 0;
+ Map<String, Integer> annotations = new HashMap<String, Integer>();
+ for (int i = 0; i < documents.size(); i++) {
+ AnnotationIndex<AnnotationFS> index = documents.get(i).getCAS().getAnnotationIndex();
+ FSIterator<AnnotationFS> iterator = index.iterator();
+ while (iterator.isValid()) {
+ AnnotationFS annotation = iterator.next();
+ if (isSlotType(annotation.getType())) {
+ TrabalAnnotation a = new TrabalAnnotation(annotation, enableFeatures);
+ if (annotations.containsKey(a.getType().getShortName())) {
+ annotations.put(a.getType().getShortName(),
+ new Integer(annotations.get(a.getType().getShortName()).intValue() + 1));
+ } else {
+ annotations.put(a.getType().getShortName(), new Integer(1));
+ }
+ // counter++;
+ }
+ }
+ for (String key : annotations.keySet()) {
+ double d = annotations.get(key).doubleValue();
+ result.put(key, new Double(1 / d));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Compares the training data to the additional data and creates a mapping of the contained
+ * errors, where the key is a combination of the error type and the names of the concerned
+ * annotations, and the values are lists of errors that can be corrected through the same type of
+ * rule.
+ *
+ * @param isFirstHalf
+ * @return
+ */
+ private Map<String, List<AnnotationError>> createErrorGroups() {
+ errors = createErrorList();
+ Collections.sort(errors);
+ Map<String, List<AnnotationError>> result = new HashMap<String, List<AnnotationError>>();
+ for (AnnotationError error : errors) {
+ String truthFeatures = "";
+ String errorFeatures = "";
+ if (error.getError() != null && error.getTruth() != null) {
+ if (enableFeatures) {
+ truthFeatures = listFeatures(error.getTruth());
+ errorFeatures = listFeatures(error.getError());
+ }
+ String key = error.getType() + "_" + error.getError().getAnnotation().getType().toString()
+ + errorFeatures + "_" + error.getTruth().getAnnotation().getType().toString()
+ + truthFeatures;
+ if (result.get(key) == null) {
+ result.put(key, new ArrayList<AnnotationError>());
+ }
+ result.get(key).add(error);
+ } else if (error.getTruth() != null) {
+ if (enableFeatures)
+ truthFeatures = listFeatures(error.getTruth());
+ String key = error.getType() + "_" + error.getTruth().getAnnotation().getType().toString()
+ + truthFeatures;
+ if (result.get(key) == null) {
+ result.put(key, new ArrayList<AnnotationError>());
+ }
+ result.get(key).add(error);
+ } else {
+ if (enableFeatures)
+ errorFeatures = listFeatures(error.getError());
+ String key = error.getType() + "_" + error.getError().getAnnotation().getType().toString()
+ + errorFeatures;
+ if (result.get(key) == null) {
+ result.put(key, new ArrayList<AnnotationError>());
+ }
+ result.get(key).add(error);
+ }
+ }
+ return result;
+ }
+
+ private String listFeatures(TextRulerExample example) {
+ String result = "";
+ for (Feature f : example.getAnnotation().getType().getFeatures()) {
+ if (!FILTERED_FEATURES.contains(f.getShortName()))
+ result += "_" + f.toString();
+ }
+ return result;
+ }
+
+ /**
+ * Creates a list of annotation pairs and the fitting type of transformation action.
+ */
+ private List<AnnotationError> createErrorList() {
+ positiveExamples = new HashMap<String, RankedList>();
+ List<AnnotationError> result = new ArrayList<AnnotationError>();
+ List<TrabalAnnotation> matches;
+ Iterator<TrabalAnnotation> iterator;
+ Iterator<TrabalAnnotation> docIterator;
+ List<TextRulerExampleDocument> documents;
+ List<TextRulerExampleDocument> goldStandard;
+ documents = additionalDocuments.getDocuments();
+ goldStandard = exampleDocuments.getDocuments();
+ for (int i = 0; i < goldStandard.size(); i++) {
+ if (shouldAbort())
+ break;
+ matches = new ArrayList<TrabalAnnotation>();
+ AnnotationIndex<AnnotationFS> index = goldStandard.get(i).getCAS().getAnnotationIndex();
+ List<TrabalAnnotation> gold = new ArrayList<TrabalAnnotation>();
+ List<TrabalAnnotation> docs = new ArrayList<TrabalAnnotation>();
+ for (AnnotationFS a : index) {
+ if (isSlotType(a.getType())) {
+ gold.add(new TrabalAnnotation(a, goldStandard.get(i), enableFeatures));
+ }
+ }
+ AnnotationIndex<AnnotationFS> docIndex = documents.get(i).getCAS().getAnnotationIndex();
+ for (AnnotationFS b : docIndex) {
+ if (isSlotType(b.getType())) {
+ docs.add(new TrabalAnnotation(b, documents.get(i), enableFeatures));
+ }
+ }
+ iterator = gold.iterator();
+ docIterator = docs.iterator();
+ TrabalAnnotation a;
+ TrabalAnnotation b;
+ // find correct annotated elements
+ int exampleIndex = 0;
+ while (iterator.hasNext()) {
+ if (shouldAbort())
+ break;
+ exampleIndex++;
+ sendStatusUpdateToDelegate("Comparing documents " + (i + 1) + " of " + goldStandard.size()
+ + ": example " + exampleIndex + " of " + gold.size(),
+ TextRulerLearnerState.ML_RUNNING, false);
+ a = iterator.next();
+ docIterator = docs.iterator();
+ while (docIterator.hasNext()) {
+ b = docIterator.next();
+ if (b.equals(a)) {
+ matches.add(a);
+ matches.add(b);
+ if (positiveExamples.containsKey(a.getType().getShortName())) {
+ RankedList list = positiveExamples.get(a.getType().getShortName());
+ list.addAll(createConditions(a));
+ positiveExamples.put(a.getType().getShortName(), list);
+ } else {
+ RankedList list = new RankedList(idf);
+ list.addAll(createConditions(a));
+ positiveExamples.put(a.getType().getShortName(), list);
+ }
+ break;
+ }
+ }
+ }
+ // create correction type errors
+ iterator = gold.iterator();
+ docIterator = docs.iterator();
+ while (iterator.hasNext()) {
+ a = iterator.next();
+ docIterator = docs.iterator();
+ while (!matches.contains(a) && docIterator.hasNext()) {
+ b = docIterator.next();
+ if (!matches.contains(b)) {
+ if (b.getBegin() == a.getBegin() && b.getEnd() == a.getEnd()) {
+ TextRulerTarget target = new TextRulerTarget(b.getType().getName(), this);
+ TextRulerExample error = new TextRulerExample(b.getDocument(), b, false, target);
+ TextRulerExample truth = new TextRulerExample(a.getDocument(), a, true, target);
+ result.add(new AnnotationError(error, truth, AnnotationErrorType.CORRECTION));
+ matches.add(a);
+ matches.add(b);
+ break;
+ }
+ }
+ }
+ }
+ // create shifting type errors
+ iterator = gold.iterator();
+ docIterator = docs.iterator();
+ List<AnnotationError> tempErrors;
+ int distance;
+ while (iterator.hasNext()) {
+ a = iterator.next();
+ docIterator = docs.iterator();
+ tempErrors = new ArrayList<AnnotationError>();
+ while (!matches.contains(a) && docIterator.hasNext()) {
+ b = docIterator.next();
+ if (!matches.contains(b)) {
+ if (b.getType().getShortName().equals(a.getType().getShortName())
+ && b.getEnd() >= a.getBegin() && b.getBegin() <= a.getEnd()) {
+ TextRulerTarget target = new TextRulerTarget(a.getType().getName(), this);
+ TextRulerExample error = new TextRulerExample(b.getDocument(), b, false, target);
+ TextRulerExample truth = new TextRulerExample(a.getDocument(), a, true, target);
+ AnnotationErrorType type;
+ if ((b.getBegin() == a.getBegin() && b.getEnd() < a.getEnd())
+ || (b.getBegin() > a.getBegin() && b.getEnd() == a.getEnd())) {
+ type = AnnotationErrorType.EXPANSION;
+ } else if (b.getBegin() > a.getBegin() || b.getEnd() > a.getEnd()) {
+ type = AnnotationErrorType.SHIFTING_LEFT;
+ } else {
+ type = AnnotationErrorType.SHIFTING_RIGHT;
+ }
+ AnnotationError err = new AnnotationError(error, truth, type);
+ if (!tempErrors.contains(err))
+ tempErrors.add(err);
+ }
+ }
+ }
+ if (tempErrors.size() > 0) {
+ AnnotationError err = tempErrors.get(0);
+ int begin = err.getError().getAnnotation().getBegin();
+ int end = err.getError().getAnnotation().getEnd();
+ distance = Math.abs(begin - a.getBegin()) + Math.abs(end - a.getEnd());
+ for (int j = 1; j < tempErrors.size(); j++) {
+ int begin2 = tempErrors.get(j).getError().getAnnotation().getBegin();
+ int end2 = tempErrors.get(j).getError().getAnnotation().getEnd();
+ if (Math.abs(begin2 - a.getBegin()) + Math.abs(end2 - a.getEnd()) < distance) {
+ distance = Math.abs(begin2 - a.getBegin()) + Math.abs(end2 - a.getEnd());
+ err = tempErrors.get(j);
+ }
+ }
+ result.add(err);
+ matches.add((TrabalAnnotation) err.getTruth().getAnnotation());
+ matches.add((TrabalAnnotation) err.getError().getAnnotation());
+ }
+ }
+ // creating shifting + correction errors
+ iterator = gold.iterator();
+ docIterator = docs.iterator();
+ while (iterator.hasNext()) {
+ a = iterator.next();
+ docIterator = docs.iterator();
+ tempErrors = new ArrayList<AnnotationError>();
+ while (!matches.contains(a) && docIterator.hasNext()) {
+ b = docIterator.next();
+ if (!matches.contains(b)) {
+ if (b.getEnd() >= a.getBegin() && b.getBegin() <= a.getEnd()) {
+ TextRulerTarget target = new TextRulerTarget(b.getType().getName(), this);
+ TextRulerExample error = new TextRulerExample(b.getDocument(), b, false, target);
+ TextRulerExample truth = new TextRulerExample(a.getDocument(), a, true, target);
+ AnnotationErrorType type;
+ if ((b.getBegin() == a.getBegin() && b.getEnd() < a.getEnd())
+ || (b.getBegin() > a.getBegin() && b.getEnd() == a.getEnd())) {
+ type = AnnotationErrorType.EXPANSION;
+ } else if (b.getBegin() > a.getBegin() || b.getEnd() > a.getEnd()) {
+ type = AnnotationErrorType.SHIFTING_LEFT;
+ } else {
+ type = AnnotationErrorType.SHIFTING_RIGHT;
+ }
+ AnnotationError err = new AnnotationError(error, truth, type);
+ if (!tempErrors.contains(err))
+ tempErrors.add(err);
+ }
+ }
+ }
+ if (tempErrors.size() > 0) {
+ AnnotationError err = tempErrors.get(0);
+ int begin = err.getError().getAnnotation().getBegin();
+ int end = err.getError().getAnnotation().getEnd();
+ distance = Math.abs(begin - a.getBegin()) + Math.abs(end - a.getEnd());
+ for (int j = 1; j < tempErrors.size(); j++) {
+ int begin2 = tempErrors.get(j).getError().getAnnotation().getBegin();
+ int end2 = tempErrors.get(j).getError().getAnnotation().getEnd();
+ if (Math.abs(begin2 - a.getBegin()) + Math.abs(end2 - a.getEnd()) < distance) {
+ distance = Math.abs(begin2 - a.getBegin()) + Math.abs(end2 - a.getEnd());
+ err = tempErrors.get(j);
+ }
+ }
+ result.add(err);
+ matches.add((TrabalAnnotation) err.getTruth().getAnnotation());
+ matches.add((TrabalAnnotation) err.getError().getAnnotation());
+ }
+ }
+ // create deleting type errors
+ docIterator = docs.iterator();
+ while (docIterator.hasNext()) {
+ b = docIterator.next();
+ if (!matches.contains(b)) {
+ TextRulerTarget target = new TextRulerTarget(b.getType().getName(), this);
+ TextRulerExample error = new TextRulerExample(b.getDocument(), b, false, target);
+ result.add(new AnnotationError(error, null, AnnotationErrorType.DELETION));
+ matches.add(b);
+ }
+ }
+ // create annotation type errors
+ iterator = gold.iterator();
+ while (iterator.hasNext()) {
+ a = iterator.next();
+ if (!matches.contains(a)) {
+ TextRulerTarget target = new TextRulerTarget(a.getType().getName(), this);
+ TextRulerExample truth = new TextRulerExample(a.getDocument(), a, true, target);
+ result.add(new AnnotationError(null, truth, AnnotationErrorType.ANNOTATION));
+ matches.add(a);
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Is called by doRun(). Creates and tests rules for a given list of error groups created by
+ * createErrorGroups().
+ *
+ * @param errorGrps
+ * @param isFirstHalf
+ * @return The best found list of correction rules.
+ */
+ private List<TrabalRule> runAlgorithm(Map<String, List<AnnotationError>> errorGrps) {
+ removeBasics();
+ List<TrabalRule> rules = new ArrayList<TrabalRule>();
+ bestRulesForStatus.clear();
+ int i = 1;
+ for (List<AnnotationError> each : errorGrps.values()) {
+ if (shouldAbort()) {
+ break;
+ }
+ Collections.sort(each);
+ String status = each.get(0).toString();
+ sendStatusUpdateToDelegate("Creating basic rules: " + status,
+ TextRulerLearnerState.ML_RUNNING, false);
+ List<TrabalRule> basicRules = removeDuplicateRules(createBasicRules(each));
+ if (basicRules.size() > maxNumberOfBasicRules) {
+ basicRules = basicRules.subList(0, maxNumberOfBasicRules);
+ }
+ sendStatusUpdateToDelegate("Testing basic rules: " + status,
+ TextRulerLearnerState.ML_RUNNING, false);
+ basicRules = testTrabalRulesOnDocumentSet(basicRules, exampleDocuments, additionalDocuments,
+ "basic rules (" + i + " of " + errorGrps.size() + ")");
+ if (basicRules.size() > 0) {
+ Collections.sort(basicRules, basicComparator);
+ bestRulesForStatus.add(basicRules.get(0));
+ }
+ result = actualResult + getRuleStrings(bestRulesForStatus);
+ sendStatusUpdateToDelegate("Testing basic rules: " + status,
+ TextRulerLearnerState.ML_RUNNING, true);
+ List<TrabalRule> learntRules = new ArrayList<TrabalRule>();
+ for (TrabalRule rule : basicRules) {
+ if (rule.getCoveringStatistics().getCoveredPositivesCount() > 0
+ && rule.getCoveringStatistics().getCoveredNegativesCount() == 0) {
+ learntRules.add(rule);
+ }
+ }
+ // Collections.sort(learntRules, basicComparator);
+ List<TrabalRule> enhancedRules = new ArrayList<TrabalRule>();
+ int rank = 1;
+ for (TrabalRule rule : basicRules) {
+ if (rule.getCoveringStatistics().getCoveredPositivesCount() > 0
+ && rule.getCoveringStatistics().getCoveredNegativesCount() > 0) {
+ if (learntRules.size() > 0) {
+ if (rule.getCoveringStatistics().getCoveredPositivesCount() > learntRules.get(0)
+ .getCoveringStatistics().getCoveredPositivesCount()) {
+ rule.setRating(rank);
+ enhancedRules.add(rule);
+ rank++;
+ }
+ } else {
+ rule.setRating(rank);
+ enhancedRules.add(rule);
+ rank++;
+ }
+ }
+ }
+ basicRules.clear();
+ try {
+ enhancedRules = enhanceRules(enhancedRules, maxNumberOfIterations, new RankedList(idf));
+ Collections.sort(enhancedRules);
+ if (enhancedRules.size() > maxNumberOfRules) {
+ enhancedRules = enhancedRules.subList(0, maxNumberOfRules);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ sendStatusUpdateToDelegate("Testing enhanced rules: " + status,
+ TextRulerLearnerState.ML_RUNNING, false);
+ enhancedRules = testTrabalRulesOnDocumentSet(enhancedRules, exampleDocuments,
+ additionalDocuments, "enhanced rules (" + i + " of " + errorGrps.size() + ")");
+ for (TrabalRule rule : enhancedRules) {
+ if (rule.getErrorRate() <= maxErrorRate) {
+ learntRules.add(rule);
+ }
+ }
+ enhancedRules.clear();
+ learntRules = removeDuplicateRules(learntRules);
+ if (learntRules.size() > 0) {
+ Collections.sort(learntRules, enhancedComparator);
+ bestRulesForStatus.remove(bestRulesForStatus.size() - 1); // TODO
+ bestRulesForStatus.add(learntRules.get(0));
+ result = actualResult + getRuleStrings(bestRulesForStatus);
+ }
+ sendStatusUpdateToDelegate("Testing optimized rules: " + status,
+ TextRulerLearnerState.ML_RUNNING, true);
+ if (learntRules.size() > maxNumberOfRules)
+ learntRules = learntRules.subList(0, maxNumberOfRules);
+ rules.addAll(learntRules);
+ i++;
+ }
+ return getBest(rules);
+ }
+
+ private List<TrabalRule> enhanceRules(List<TrabalRule> learntRules, int iterations,
+ RankedList conditions) throws Exception {
+ if (shouldAbort() || learntRules.size() == 0)
+ return learntRules;
+ sendStatusUpdateToDelegate("Adding conditions to rules (" + iterations + ". Iteration)... ",
+ TextRulerLearnerState.ML_RUNNING, false);
+ if (iterations > 0) {
+ List<TrabalRule> rules = new ArrayList<TrabalRule>();
+ List<TrabalRule> newRules = new ArrayList<TrabalRule>();
+ if (conditions.size() == 0)
+ conditions = createConditions(learntRules);
+ for (int i = 0; i < learntRules.size(); i++) {
+ rules.add(learntRules.get(i));
+ for (int j = 0; j < conditions.size() && j < 50; j++) {
+ TrabalRule newRule = learntRules.get(i).copy();
+ if (!newRule.getConditions().contains(conditions.get(j))) {
+ newRule.addCondition(conditions.get(j), (j + 1));
+ newRules.add(newRule);
+ }
+ }
+ }
+ rules.addAll(enhanceRules(newRules, iterations - 1, conditions));
+ return rules;
+ }
+ return learntRules;
+ }
+
+ // private List<Condition> getBestConditions(List<Condition> negExampleConditions,
+ // List<Condition> posExampleConditions) {
+ // List<Condition> result = new ArrayList<Condition>();
+ // RankedList<Condition> negExampleCounter = new RankedList<Condition>();
+ // RankedList<Condition> posExampleCounter = new RankedList<Condition>();
+ // Map<Condition, Double> conditionValues = new HashMap<Condition, Double>();
+ // negExampleCounter.addAll(negExampleConditions);
+ // posExampleCounter.addAll(posExampleConditions);
+ // for (Condition c : negExampleConditions) {
+ // int p = 1;
+ // if (posExampleCounter.containsKey(c)) {
+ // p += posExampleCounter.get(c).intValue();
+ // }
+ // conditionValues.put(c, new Double(negExampleCounter.get(c).doubleValue() / p));
+ // }
+ // while (conditionValues.size() > 0) {
+ // Double max = 0.0;
+ // Condition c = null;
+ // for (int i = 0; i < conditionValues.size(); i++) {
+ // if (conditionValues.get(i) > max) {
+ // max = conditionValues.get(i);
+ // c = (Condition) conditionValues.keySet().toArray()[i];
+ // }
+ // }
+ // if (max < 1 || c == null)
+ // break;
+ // result.add(c);
+ // conditionValues.remove(c);
+ // }
+ // return result;
+ // }
+
+ // /**
+ // * Takes a list of rules with conditions and chooses those, which are expected to create the
+ // best
+ // * results.
+ // *
+ // * @param rules
+ // * @return
+ // */
+ // private List<TrabalRule> getBestConditions(List<TrabalRule> rules) {
+ // sendStatusUpdateToDelegate("Choosing conditions... ", TextRulerLearnerState.ML_RUNNING, false);
+ // if (rules.size() < maxNumberOfRules)
+ // return rules;
+ // List<TrabalRule> result = new ArrayList<TrabalRule>();
+ // Map<String, Integer> distribution = new HashMap<String, Integer>();
+ // for (TrabalRule rule : rules) {
+ // String key = "";
+ // List<Condition> conditions = rule.getConditions();
+ // Collections.sort(conditions);
+ // for (Condition c : conditions) {
+ // key += c;
+ // }
+ // if (distribution.containsKey(key)) {
+ // distribution.put(key, new Integer(distribution.get(key).intValue() + 1));
+ // } else {
+ // distribution.put(key, new Integer(1));
+ // }
+ // }
+ // List<TrabalRule> newRules;
+ // while (result.size() < maxNumberOfRules && distribution.size() > 0) {
+ // if (shouldAbort())
+ // break;
+ // int max = 0;
+ // String maxKey = "";
+ // for (String key : distribution.keySet()) {
+ // if (distribution.get(key).intValue() > max) {
+ // max = distribution.get(key).intValue();
+ // maxKey = key;
+ // }
+ // }
+ // distribution.remove(maxKey);
+ // newRules = new ArrayList<TrabalRule>();
+ // TrabalRule example = null;
+ // for (TrabalRule rule : rules) {
+ // String key = "";
+ // List<Condition> conditions = rule.getConditions();
+ // Collections.sort(conditions);
+ // for (Condition c : conditions) {
+ // key += c;
+ // }
+ // if (distribution.containsKey(key)) {
+ // if (distribution.get(key).intValue() == max) {
+ // result.add(rule);
+ // if (example == null)
+ // example = rule;
+ // }
+ // }
+ // }
+ // if (example != null) {
+ // for (TrabalRule rule : rules) {
+ // if (!rule.hasSameBasicRule(example)) {
+ // newRules.add(rule);
+ // }
+ // }
+ // }
+ // rules = newRules;
+ // }
+ // if (result.size() > maxNumberOfRules)
+ // return result.subList(0, maxNumberOfRules);
+ // return result;
+ // }
+
+ /**
+ * Chooses the best final rules from the results of runAlgorithm().
+ *
+ * @param rules
+ * @param isFirstHalf
+ * @return A non redundant list of correction rules.
+ */
+ private List<TrabalRule> getBest(List<TrabalRule> rules) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ Collections.sort(rules, enhancedComparator);
+ while (rules.size() > 0) {
+ List<TrabalRule> testRules = new ArrayList<TrabalRule>();
+ List<TrabalRule> newRules = new ArrayList<TrabalRule>();
+ TrabalRule bestRule = rules.get(0);
+ if (bestRule.getErrorRate() > maxErrorRate) {
+ rules.remove(0);
+ continue;
+ }
+ if (bestRule.getCoveringStatistics().getCoveredPositivesCount() == 0) {
+ break;
+ }
+ sendStatusUpdateToDelegate("Chosen rule: " + bestRule, TextRulerLearnerState.ML_RUNNING,
+ false);
+ result.add(bestRule);
+ rules.remove(0);
+ Set<TextRulerExample> coveredExamples = bestRule.getCoveringStatistics()
+ .getCoveredPositiveExamples();
+ for (TrabalRule rule : rules) {
+ TextRulerStatisticsCollector collector = rule.getCoveringStatistics();
+ Iterator<TextRulerExample> iterator = coveredExamples.iterator();
+ while (iterator.hasNext()) {
+ TextRulerExample example = iterator.next();
+ if (collector.getCoveredPositiveExamples().contains(example)) {
+ collector.getCoveredPositiveExamples().remove(example);
+ // if (rule.getErrorType() == ErrorType.ANNOTATION
+ // || rule.getErrorType() == ErrorType.DELETING) {
+ // collector.incCoveredPositives(-1);
+ // } else {
+ collector.incCoveredPositives(-1);
+ // }
+ }
+ if (collector.getCoveredPositivesCount() <= 0) {
+ break;
+ }
+ }
+ if (rule.getErrorRate() > maxErrorRate
+ || rule.getCoveringStatistics().getCoveredPositivesCount() <= 0) {
+ if (bestRulesForStatus.contains(rule))
+ bestRulesForStatus.remove(rule);
+ this.result = actualResult + getRuleStrings(bestRulesForStatus);
+ continue;
+ } else if (rule.contains(bestRule.getAnnotation())
+ || rule.contains(bestRule.getTargetAnnotation())) {
+ testRules.add(rule.copy());
+ } else {
+ newRules.add(rule);
+ }
+ }
+ rules = new ArrayList<TrabalRule>();
+ updateDocumentData(bestRule);
+ removeBasics();
+ testRules = testTrabalRulesOnDocumentSet(testRules, exampleDocuments, additionalDocuments,
+ "final rules ");
+ for (TrabalRule rule : testRules) {
+ if (rule.getErrorRate() <= maxErrorRate
+ && rule.getCoveringStatistics().getCoveredPositivesCount() > 0) {
+ rules.add(rule);
+ }
+ }
+ rules.addAll(newRules);
+ Collections.sort(rules, enhancedComparator);
+ }
+ this.result = actualResult;
+ return result;
+ }
+
+ /**
+ * Is called from getBest(). Applies the chosen rules to the additional documents, so the learner
+ * can ignore the already corrected errors in the next interation.
+ *
+ * @param rule
+ */
+ private void updateDocumentData(TrabalRule rule) {
+ try {
+ sendStatusUpdateToDelegate("Writing rules...", TextRulerLearnerState.ML_RUNNING, false);
+ rule.saveToRulesFile(getTempRulesFileName());
+ for (TextRulerExampleDocument doc : additionalDocuments.getDocuments()) {
+ ae.process(doc.getCAS());
+ }
+ if (bestRulesForStatus.contains(rule)) {
+ bestRulesForStatus.remove(rule);
+ }
+ actualResult += rule.toString();
+ result = actualResult + getRuleStrings(bestRulesForStatus);
+ sendStatusUpdateToDelegate("Writing rules...", TextRulerLearnerState.ML_RUNNING, true);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Creates basic rules for the given types of errors.
+ *
+ * @param errors
+ * @return
+ */
+ public List<TrabalRule> createBasicRules(List<AnnotationError> errors) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ for (AnnotationError each : errors) {
+ switch (each.getType()) {
+ case SHIFTING_LEFT:
+ case SHIFTING_RIGHT:
+ case EXPANSION:
+ result.addAll(createShiftingRules(each));
+ break;
+ case CORRECTION:
+ result.addAll(createCorrectionRules(each));
+ break;
+ case ANNOTATION:
+ result.addAll(createAnnotationRules(each));
+ break;
+ case DELETION:
+ result.addAll(createDeletionRules(each));
+ break;
+ }
+ }
+ return getBestBasicRule(result);
+ }
+
+ /**
+ * Chooses the rules which are expected to be the best.
+ *
+ * @param rules
+ * @return
+ */
+ private List<TrabalRule> getBestBasicRule(List<TrabalRule> rules) {
+ if (rules.size() < maxNumberOfBasicRules)
+ return rules;
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ sendStatusUpdateToDelegate("Choosing basic rules... ", TextRulerLearnerState.ML_RUNNING, false);
+ Map<Integer, Integer> distribution = new HashMap<Integer, Integer>();
+ for (TrabalRule rule : rules) {
+ Integer key = new Integer(rule.hashCode());
+ if (distribution.containsKey(key)) {
+ distribution.put(key, new Integer(distribution.get(key).intValue() + 1));
+ } else {
+ distribution.put(key, new Integer(1));
+ }
+ }
+ List<TrabalRule> newRules;
+ while (result.size() < maxNumberOfBasicRules && distribution.size() > 0) {
+ if (shouldAbort())
+ break;
+ int max = 0;
+ Integer maxKey = new Integer(0);
+ for (Integer key : distribution.keySet()) {
+ if (distribution.get(key).intValue() > max) {
+ max = distribution.get(key).intValue();
+ maxKey = key;
+ }
+ }
+ distribution.remove(maxKey);
+ newRules = new ArrayList<TrabalRule>();
+ TrabalRule example = null;
+ for (TrabalRule rule : rules) {
+ Integer key = new Integer(rule.hashCode());
+ if (distribution.containsKey(key)) {
+ if (distribution.get(key).intValue() == max) {
+ result.add(rule);
+ if (example == null)
+ example = rule;
+ }
+ }
+ }
+ for (TrabalRule rule : rules) {
+ newRules.add(rule);
+ }
+ rules = newRules;
+ }
+ if (result.size() > maxNumberOfBasicRules)
+ return result.subList(0, maxNumberOfBasicRules);
+ return result;
+ }
+
+ /**
+ * Creates basic rules for shifting type errors.
+ *
+ * @param each
+ * @return
+ */
+ private List<TrabalRule> createShiftingRules(AnnotationError each) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ TextRulerExample error = each.getError();
+ TextRulerExample truth = each.getTruth();
+ int errorBegin = error.getAnnotation().getBegin();
+ int errorEnd = error.getAnnotation().getEnd();
+ int truthBegin = truth.getAnnotation().getBegin();
+ int truthEnd = truth.getAnnotation().getEnd();
+
+ if ((errorBegin > truthBegin && errorEnd > truthEnd) // shift both borders to left
+ || (errorBegin < truthBegin && errorEnd < truthEnd)) { // shift both borders to right
+ result.addAll(createShiftAllRules(error, truth));
+ } else if (errorBegin > truthBegin // expand to left
+ || errorEnd < truthEnd) { // expand to right
+ result.addAll(createExpansionRules(error, truth));
+ } else {
+ result.addAll(createShiftAllRules(error, truth)); // reduce the annotation
+ }
+ return result;
+ }
+
+ /**
+ * Creates basic rules for shifting errors, where both borders are shiftet to the same side or the
+ * size of the annotation is reduced.
+ *
+ * @param error
+ * The error example
+ * @param truth
+ * The truth example
+ */
+ private List<TrabalRule> createShiftAllRules(TextRulerExample error, TextRulerExample truth) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ TextRulerExampleDocument document = error.getDocument();
+ CAS cas = error.getDocumentCAS();
+ List<TrabalRuleItem> truthLeftBorder = getBorderOfExample(truth, document, cas, true);
+ List<TrabalRuleItem> truthRightBorder = getBorderOfExample(truth, document, cas, false);
+ List<TrabalRuleItem> errorLeftBorder = getBorderOfExample(error, document, cas, true);
+ TextRulerTarget target = error.getTarget();
+ TrabalAnnotation errorAnnotation = (TrabalAnnotation) error.getAnnotation();
+ TrabalAnnotation truthAnnotation = (TrabalAnnotation) truth.getAnnotation();
+ // boolean shiftToLeft = error.getAnnotation().getBegin() > truth.getAnnotation().getBegin();
+ // if (shiftToLeft) {
+ for (TrabalRuleItem a : truthLeftBorder) {
+ for (TrabalRuleItem b : errorLeftBorder) {
+ for (TrabalRuleItem c : truthRightBorder) {
+ ShiftAllRule newRule;
+ if (truth.getAnnotation().getEnd() < error.getAnnotation().getEnd())
+ newRule = new ShiftAllRule(this, target, errorAnnotation, truthAnnotation,
+ AnnotationErrorType.SHIFTING_LEFT);
+ else
+ newRule = new ShiftAllRule(this, target, errorAnnotation, truthAnnotation,
+ AnnotationErrorType.SHIFTING_RIGHT);
+ newRule.setFrontBoundaryItem(a);
+ if (b.getAnnotation().getBegin() != a.getAnnotation().getBegin()) {
+ newRule.setErrorBoundaryItem(b);
+ }
+ newRule.setRearBoundaryItem(c);
+ result.add(newRule);
+ }
+ }
+ }
+ // } else {
+ // for (TrabalRuleItem a : errorLeftBorder) {
+ // for (TrabalRuleItem b : truthLeftBorder) {
+ // for (TrabalRuleItem c : truthRightBorder) {
+ // ShiftingRule newRule = new ShiftingRule(this, target, errorAnnotation, truthAnnotation,
+ // AnnotationErrorType.SHIFTING_RIGHT);
+ // newRule.addFillerItem(a);
+ // if (b.getAnnotation().getBegin() != a.getAnnotation().getBegin()) {
+ // newRule.addFillerItem(b);
+ // }
+ // newRule.addFillerItem(c);
+ // result.add(newRule);
+ // }
+ // }
+ // }
+ // }
+ return result;
+ }
+
+ /**
+ * Creates basic rules for expansion type errors.
+ *
+ * @param error
+ * @param truth
+ * @return
+ */
+ private List<TrabalRule> createExpansionRules(TextRulerExample error, TextRulerExample truth) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ TextRulerExampleDocument document = error.getDocument();
+ CAS cas = error.getDocumentCAS();
+ boolean shiftToLeft = error.getAnnotation().getBegin() > truth.getAnnotation().getBegin();
+ List<TrabalRuleItem> border;
+ if (shiftToLeft) {
+ border = getBorderOfExample(truth, document, cas, true);
+ } else {
+ border = getBorderOfExample(truth, document, cas, false);
+ }
+ for (TrabalRuleItem item : border) {
+ ExpansionRule newRule = new ExpansionRule(this, error.getTarget(),
+ (TrabalAnnotation) error.getAnnotation(), (TrabalAnnotation) truth.getAnnotation(),
+ AnnotationErrorType.EXPANSION);
+ if (shiftToLeft)
+ newRule.setFrontBoundaryItem(item);
+ else
+ newRule.setRearBoundaryItem(item);
+ if (!result.contains(newRule))
+ result.add(newRule);
+ }
+ return result;
+ }
+
+ /**
+ * Creates basic rules for annotation type errors.
+ *
+ * @param each
+ * @return
+ */
+ private List<TrabalRule> createAnnotationRules(AnnotationError each) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ TextRulerExample truth = each.getTruth();
+ TextRulerExampleDocument document = additionalDocuments.getDocuments().get(
+ exampleDocuments.getDocuments().indexOf(truth.getDocument()));
+ CAS cas = document.getCAS();
+ List<TrabalRuleItem> truthLeftBorder = getBorderOfExample(truth, document, cas, true);
+ List<TrabalRuleItem> truthRightBorder = getBorderOfExample(truth, document, cas, false);
+ TextRulerTarget target = truth.getTarget();
+ TrabalAnnotation truthAnnotation = (TrabalAnnotation) truth.getAnnotation();
+ for (TrabalRuleItem front : truthLeftBorder) {
+ for (TrabalRuleItem rear : truthRightBorder) {
+ AnnotationRule newRule = new AnnotationRule(this, target, truthAnnotation);
+ newRule.setFrontBoundary(front);
+ newRule.setRearBoundary(rear);
+ if (!result.contains(newRule))
+ result.add(newRule);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Creates basic rules for deletion type errors.
+ *
+ * @param each
+ * @return
+ */
+ private List<TrabalRule> createDeletionRules(AnnotationError each) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ TextRulerExample error = each.getError();
+ TrabalRule blankRule = new DeletionRule(this, error.getTarget(),
+ (TrabalAnnotation) error.getAnnotation());
+ result.add(blankRule);
+ return result;
+ }
+
+ /**
+ * Creates basic rules for correction type errors.
+ *
+ * @param each
+ * @return
+ */
+ private List<TrabalRule> createCorrectionRules(AnnotationError each) {
+ List<TrabalRule> result = new ArrayList<TrabalRule>();
+ TextRulerExample error = each.getError();
+ TextRulerExample truth = each.getTruth();
+ TrabalRule blankRule = new CorrectionRule(this, error.getTarget(),
+ (TrabalAnnotation) error.getAnnotation(), (TrabalAnnotation) truth.getAnnotation());
+ result.add(blankRule);
+ return result;
+ }
+
+ /**
+ * Examines the border of an example annotation and returns all annotations before and on that
+ * border if looking at the left border or returns all annotations on and after the example if
+ * looking at the right border.
+ *
+ * @param example
+ * The example that should be examined
+ * @param examineLeftBorder
+ * set true for the left border or false for the right border to be examined
+ */
+ private List<TrabalRuleItem> getBorderOfExample(TextRulerExample example,
+ TextRulerExampleDocument document, CAS cas, boolean examineLeftBorder) {
+ List<TrabalRuleItem> result = new ArrayList<TrabalRuleItem>();
+ int begin = example.getAnnotation().getBegin();
+ int end = example.getAnnotation().getEnd();
+ TrabalRuleItem ruleItem = new TrabalRuleItem(begin, end, cas, enableFeatures);
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, begin, end);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().subiterator(pointer);
+ if (examineLeftBorder) {
+ result.addAll(getTermsBefore(ruleItem, document));
+ iterator.moveToFirst();
+ while (iterator.isValid()) {
+ FeatureStructure fs = iterator.get();
+ AnnotationFS a = (AnnotationFS) fs;
+ if (a.getBegin() == begin && a.getEnd() < end) {
+ if (!filterSet.contains(a.getType().getName())
+ && !a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ result.add(new TrabalRuleItem(new TrabalAnnotation(a, document, enableFeatures)));
+ }
+ } else {
+ if (a.getBegin() > begin) {
+ break;
+ }
+ }
+ iterator.moveToNext();
+ }
+ } else {
+ iterator.moveToLast();
+ while (iterator.isValid()) {
+ FeatureStructure fs = iterator.get();
+ AnnotationFS a = (AnnotationFS) fs;
+ if (a.getBegin() > begin && a.getEnd() == end) {
+ if (!filterSet.contains(a.getType().getName())
+ && !a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ result.add(new TrabalRuleItem(new TrabalAnnotation(a, document, enableFeatures)));
+ }
+ } else {
+ break;
+ }
+ iterator.moveToPrevious();
+ }
+ result.addAll(getTermsAfter(ruleItem, document));
+ }
+ return result;
+ }
+
+ /**
+ * Creates conditions for a given rule.
+ *
+ * @param rule
+ * @return
+ */
+ public RankedList createConditions(List<TrabalRule> rules) {
+ if (rules.size() == 0)
+ return new RankedList(idf);
+ RankedList result;
+ RankedList error = new RankedList(idf);
+ RankedList truth = new RankedList(idf);
+ switch (rules.get(0).getErrorType()) {
+ case SHIFTING_LEFT:
+ case SHIFTING_RIGHT:
+ case EXPANSION:
+ for (TrabalRule rule : rules) {
+ error.addAll(createConditions(rule.getAnnotation()));
+ truth.addAll(createConditions(rule.getTargetAnnotation()));
+ }
+ result = error.subtract(truth.unite(getPositiveExamplesFor(rules.get(0)
+ .getTargetAnnotation().getType())));
+ setNegative(result);
+ result.addAll(truth.cut(
+ getPositiveExamplesFor(rules.get(0).getTargetAnnotation().getType())).subtract(
+ error));
+ return result;
+ case CORRECTION:
+ for (TrabalRule rule : rules) {
+ truth.addAll(createConditions(rule.getAnnotation()));
+ }
+ result = getPositiveExamplesFor(rules.get(0).getAnnotation().getType()).subtract(
+ truth.unite(getPositiveExamplesFor(rules.get(0).getTargetAnnotation().getType())));
+ setNegative(result);
+ if (getPositiveExamplesFor(rules.get(0).getTargetAnnotation().getType()).size() > 0) {
+ result.addAll(truth.cut(
+ getPositiveExamplesFor(rules.get(0).getTargetAnnotation().getType())).subtract(
+ getPositiveExamplesFor(rules.get(0).getAnnotation().getType())));
+ } else {
+ result.addAll(truth);
+ }
+ return result;
+ case ANNOTATION:
+ for (TrabalRule rule : rules) {
+ truth.addAll(createConditions(rule.getFrontBoundary(), rule.getRearBoundary(),
+ rule.getTargetAnnotation()));
+ }
+ if (getPositiveExamplesFor(rules.get(0).getTargetAnnotation().getType()).size() > 0) {
+ return truth.cut(getPositiveExamplesFor(rules.get(0).getTargetAnnotation().getType()));
+ }
+ return truth;
+ case DELETION:
+ for (TrabalRule rule : rules) {
+ error.addAll(createConditions(rule.getAnnotation()));
+ }
+ result = getPositiveExamplesFor(rules.get(0).getAnnotation().getType()).subtract(error);
+ setNegative(result);
+ result.addAll(createConditions(rules.get(0).getAnnotation()).subtract(
+ getPositiveExamplesFor(rules.get(0).getAnnotation().getType())));
+ return result;
+ }
+ return null;
+ }
+
+ public RankedList getPositiveExamplesFor(Type type) {
+ if (positiveExamples.containsKey(type)) {
+ return positiveExamples.get(type);
+ } else {
+ return new RankedList(idf);
+ }
+ }
+
+ public void setNegative(RankedList list) {
+ for (Condition element : list.getRanking().keySet()) {
+ element.setNegative();
+ }
+ for (Condition element : list) {
+ element.setNegative();
+ }
+ }
+
+ /**
+ * Creates conditions for a rule given by its boundaries and annotation type.
+ *
+ * @param frontBoundary
+ * @param rearBoundary
+ * @param truth
+ * @return
+ */
+ private RankedList createConditions(TrabalRuleItem frontBoundary, TrabalRuleItem rearBoundary,
+ TrabalAnnotation truth) {
+ RankedList result = new RankedList(idf);
+ TextRulerExampleDocument doc = additionalDocuments.getDocuments().get(
+ exampleDocuments.getDocuments().indexOf(truth.getDocument()));
+ CAS cas = doc.getCAS();
+ for (TrabalRuleItem item : getTermsBefore(frontBoundary, doc)) {
+ result.add(new Condition(ConditionType.AFTER, item));
+ }
+ for (TrabalRuleItem item : getTermsAfter(rearBoundary, doc)) {
+ result.add(new Condition(ConditionType.BEFORE, item));
+ }
+ for (TrabalRuleItem item : getConsumingTerms(frontBoundary, doc)) {
+ if (!item.getAnnotation().equals(cas.getDocumentAnnotation())) {
+ result.add(new Condition(ConditionType.PARTOF, item));
+ }
+ }
+ if (frontBoundary.getAnnotation().getBegin() < truth.getBegin()) {
+ for (TrabalRuleItem item : getTermsAfter(frontBoundary, doc)) {
+ result.add(new Condition(ConditionType.STARTSWITH, item));
+ }
+ } else {
+ for (List<TrabalRuleItem> list : getFirstTermsWithinBounds(frontBoundary.getAnnotation()
+ .getBegin(), rearBoundary.getAnnotation().getEnd(), doc, cas, 1)) {
+ result.add(new Condition(ConditionType.STARTSWITH, list.get(0)));
+ }
+ }
+ if (rearBoundary.getAnnotation().getEnd() > truth.getEnd()) {
+ for (TrabalRuleItem item : getTermsBefore(rearBoundary, doc)) {
+ result.add(new Condition(ConditionType.ENDSWITH, item));
+ }
+ } else {
+ for (List<TrabalRuleItem> list : getLastTermsWithinBounds(rearBoundary.getAnnotation()
+ .getBegin(), frontBoundary.getAnnotation().getEnd(), doc, cas, 1)) {
+ result.add(new Condition(ConditionType.ENDSWITH, list.get(0)));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Creates conditions for a rule, given by its annotation.
+ *
+ * @param error
+ * @return
+ */
+ private RankedList createConditions(TrabalAnnotation annotation) {
+ RankedList result = new RankedList(idf);
+ TrabalRuleItem ruleItem = new TrabalRuleItem(annotation);
+ TextRulerExampleDocument doc = annotation.getDocument();
+ CAS cas = doc.getCAS();
+ for (TrabalRuleItem item : getTermsBefore(ruleItem, doc)) {
+ result.add(new Condition(ConditionType.AFTER, item));
+ }
+ for (TrabalRuleItem item : getTermsAfter(ruleItem, doc)) {
+ result.add(new Condition(ConditionType.BEFORE, item));
+ }
+ for (TrabalRuleItem item : getConsumingTerms(ruleItem, doc)) {
+ if (!item.getAnnotation().equals(cas.getDocumentAnnotation())) {
+ result.add(new Condition(ConditionType.PARTOF, item));
+ }
+ }
+ for (List<TrabalRuleItem> list : getFirstTermsWithinBounds(ruleItem.getAnnotation().getBegin(),
+ ruleItem.getAnnotation().getEnd(), doc, cas, 1)) {
+ result.add(new Condition(ConditionType.STARTSWITH, list.get(0)));
+ }
+ for (List<TrabalRuleItem> list : getLastTermsWithinBounds(ruleItem.getAnnotation().getBegin(),
+ ruleItem.getAnnotation().getEnd(), doc, cas, 1)) {
+ result.add(new Condition(ConditionType.ENDSWITH, list.get(0)));
+ }
+ for (TrabalRuleItem item : getSingleTermsWithinBounds(ruleItem.getAnnotation().getBegin(),
+ ruleItem.getAnnotation().getEnd(), doc, cas)) {
+ result.add(new Condition(ConditionType.CONTAINS, item));
+ }
+ return result;
+ }
+
+ /**
+ * Returns all items directly preceding the given example rule item.
+ *
+ * @param ruleItem
+ * Examined rule item
+ * @param document
+ * Example document
+ * @param cas
+ * The CAS
+ * @return List of rule items that precede the given example
+ */
+ private List<TrabalRuleItem> getTermsBefore(TrabalRuleItem ruleItem,
+ TextRulerExampleDocument document) {
+ List<TrabalRuleItem> result = new ArrayList<TrabalRuleItem>();
+ int begin = ruleItem.getAnnotation().getBegin();
+ CAS cas = document.getCAS();
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, begin, begin);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().iterator(pointer);
+ int nextEnd = -1;
+
+ while (iterator.isValid()) {
+ FeatureStructure fs = iterator.get();
+ if (fs instanceof AnnotationFS) {
+ AnnotationFS a = (AnnotationFS) fs;
+ if (!filterSet.contains(a.getType().getName())
+ && !a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ if (a.getEnd() > begin) {
+ iterator.moveToPrevious();
+ continue;
+ }
+ if (nextEnd == -1) {
+ nextEnd = a.getEnd();
+ }
+ if (a.getEnd() >= nextEnd && a.getEnd() <= begin) {
+ TrabalRuleItem term = new TrabalRuleItem(new TrabalAnnotation(a, document,
+ enableFeatures));
+ result.add(term);
+ }
+ }
+ }
+ iterator.moveToPrevious();
+ }
+ return result;
+ }
+
+ /**
+ * Returns all items directly following the given example rule item.
+ *
+ * @param ruleItem
+ * Examined rule item
+ * @param document
+ * Example document
+ * @param cas
+ * The CAS
+ * @return List of rule items that follow the given example
+ */
+ private List<TrabalRuleItem> getTermsAfter(TrabalRuleItem ruleItem,
+ TextRulerExampleDocument document) {
+ List<TrabalRuleItem> result = new ArrayList<TrabalRuleItem>();
+ int end = ruleItem.getAnnotation().getEnd();
+ CAS cas = document.getCAS();
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, end, Integer.MAX_VALUE);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().iterator(pointer);
+ int nextBegin = -1;
+ while (iterator.isValid()) {
+ FeatureStructure fs = iterator.get();
+ if (fs instanceof AnnotationFS) {
+ AnnotationFS a = (AnnotationFS) fs;
+ if (!filterSet.contains(a.getType().getName())
+ && !a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ if (nextBegin == -1) {
+ nextBegin = a.getBegin();
+ }
+ if (a.getBegin() <= nextBegin && a.getBegin() >= end) {
+ TrabalRuleItem term = new TrabalRuleItem(new TrabalAnnotation(a, document,
+ enableFeatures));
+ result.add(term);
+ }
+ }
+ }
+ iterator.moveToNext();
+ }
+ return result;
+ }
+
+ /**
+ * Gets the left border within the bounds, that means a list of elements that start with the
+ * border and those that end right before it.
+ *
+ * @param startPos
+ * @param endPos
+ * @param document
+ * @param cas
+ * @param numberOfSideItems
+ * @return
+ */
+ private List<List<TrabalRuleItem>> getFirstTermsWithinBounds(int startPos, int endPos,
+ TextRulerExampleDocument document, CAS cas, int numberOfSideItems) {
+ List<List<TrabalRuleItem>> preItems = new ArrayList<List<TrabalRuleItem>>();
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, startPos, endPos);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().iterator(pointer);
+ List<AnnotationFS> startAs = new ArrayList<AnnotationFS>();
+ int firstBegin = -1;
+ while (iterator.isValid()) {
+ FeatureStructure fs = iterator.get();
+ AnnotationFS a = (AnnotationFS) fs;
+
+ if (a.getBegin() >= startPos && a.getEnd() <= endPos) {
+ if (!filterSet.contains(a.getType().getName())
+ && !a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ if (firstBegin == -1) {
+ firstBegin = a.getBegin();
+ }
+ if (a.getBegin() == firstBegin)
+ startAs.add(a);
+ }
+ iterator.moveToNext();
+ } else {
+ iterator.moveToNext();
+ }
+ }
+
+ for (AnnotationFS each : startAs) {
+ List<TrabalRuleItem> startList = new ArrayList<TrabalRuleItem>();
+ TrabalRuleItem term = new TrabalRuleItem(new TrabalAnnotation(each, document, enableFeatures));
+ startList.add(term);
+ preItems.add(startList);
+ }
+ preItems = addFollowing(preItems, endPos, document, cas, 1, numberOfSideItems);
+ return preItems;
+ }
+
+ /**
+ * Gets the right border within the bounds, that means a list of elements that end with the border
+ * and those that start right behind it.
+ *
+ * @param startPos
+ * @param endPos
+ * @param document
+ * @param cas
+ * @param numberOfSideItems
+ * @return
+ */
+ private List<List<TrabalRuleItem>> getLastTermsWithinBounds(int startPos, int endPos,
+ TextRulerExampleDocument document, CAS cas, int numberOfSideItems) {
+ List<List<TrabalRuleItem>> postItems = new ArrayList<List<TrabalRuleItem>>();
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, startPos, endPos);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().iterator(pointer);
+ iterator.moveToLast();
+ List<AnnotationFS> endAs = new ArrayList<AnnotationFS>();
+ int lastEnd = -1;
+ while (iterator.isValid()) {
+ FeatureStructure fs = iterator.get();
+ AnnotationFS a = (AnnotationFS) fs;
+
+ if (a.getEnd() <= endPos && a.getBegin() >= startPos) {
+ if (!filterSet.contains(a.getType().getName())
+ && !a.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ if (lastEnd == -1) {
+ lastEnd = a.getEnd();
+ }
+ if (a.getEnd() == lastEnd)
+ endAs.add(a);
+ }
+ iterator.moveToPrevious();
+ } else {
+ iterator.moveToPrevious();
+ }
+ }
+
+ for (AnnotationFS each : endAs) {
+ List<TrabalRuleItem> endList = new ArrayList<TrabalRuleItem>();
+ TrabalRuleItem term = new TrabalRuleItem(new TrabalAnnotation(each, document, enableFeatures));
+ endList.add(term);
+ postItems.add(endList);
+ }
+ postItems = addPreceding(postItems, startPos, document, cas, 1, numberOfSideItems);
+ return postItems;
+ }
+
+ /**
+ * Returns a list of rule elements that precede the given elements.
+ *
+ * @param lists
+ * @param till
+ * @param document
+ * @param cas
+ * @param index
+ * @param maxNumberOfItems
+ * @return
+ */
+ private List<List<TrabalRuleItem>> addPreceding(List<List<TrabalRuleItem>> lists, int till,
+ TextRulerExampleDocument document, CAS cas, int index, int maxNumberOfItems) {
+ if (index >= maxNumberOfItems) {
+ return lists;
+ }
+ List<List<TrabalRuleItem>> result = new ArrayList<List<TrabalRuleItem>>();
+ for (List<TrabalRuleItem> list : lists) {
+ if (shouldAbort()) {
+ break;
+ }
+ List<List<TrabalRuleItem>> newLists = new ArrayList<List<TrabalRuleItem>>();
+ TrabalRuleItem first = list.get(0);
+ List<TrabalRuleItem> termsBefore = getTermsBefore(first, document);
+ if (termsBefore.isEmpty()) {
+ continue;
+ }
+ for (TrabalRuleItem eachBefore : termsBefore) {
+ if (eachBefore.getAnnotation().getBegin() >= till) {
+ List<TrabalRuleItem> newList = new ArrayList<TrabalRuleItem>();
+ newList.addAll(list);
+ newList.add(0, eachBefore);
+ newLists.add(newList);
+ newLists = addPreceding(newLists, till, document, cas, index + 1, maxNumberOfItems);
+ }
+ }
+ result.addAll(newLists);
+ }
+ if (result.isEmpty()) {
+ return lists;
+ }
+ return result;
+ }
+
+ /**
+ * Returns a list of rule elements that follow on the given elements.
+ *
+ * @param lists
+ * @param till
+ * @param document
+ * @param cas
+ * @param index
+ * @param maxNumberOfItems
+ * @return
+ */
+ private List<List<TrabalRuleItem>> addFollowing(List<List<TrabalRuleItem>> lists, int till,
+ TextRulerExampleDocument document, CAS cas, int index, int maxNumberOfItems) {
+ if (index >= maxNumberOfItems) {
+ return lists;
+ }
+ List<List<TrabalRuleItem>> result = new ArrayList<List<TrabalRuleItem>>();
+ for (List<TrabalRuleItem> list : lists) {
+ if (shouldAbort()) {
+ break;
+ }
+ List<List<TrabalRuleItem>> newLists = new ArrayList<List<TrabalRuleItem>>();
+ TrabalRuleItem last = list.get(list.size() - 1);
+ List<TrabalRuleItem> termsAfter = getTermsAfter(last, document);
+ if (termsAfter.isEmpty()) {
+ continue;
+ }
+ for (TrabalRuleItem eachAfter : termsAfter) {
+ if (eachAfter.getAnnotation().getEnd() <= till) {
+ List<TrabalRuleItem> newList = new ArrayList<TrabalRuleItem>();
+ newList.addAll(list);
+ newList.add(eachAfter);
+ newLists.add(newList);
+ newLists = addFollowing(newLists, till, document, cas, index + 1, maxNumberOfItems);
+ }
+ }
+ result.addAll(newLists);
+ }
+ if (result.isEmpty()) {
+ return lists;
+ }
+ return result;
+ }
+
+ /**
+ * Returns a list of all items, consuming the given RuleItem.
+ */
+ private List<TrabalRuleItem> getConsumingTerms(TrabalRuleItem ruleItem,
+ TextRulerExampleDocument document) {
+ CAS cas = document.getCAS();
+ List<TrabalRuleItem> result = new ArrayList<TrabalRuleItem>();
+ int begin = ruleItem.getAnnotation().getBegin();
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, 0, begin);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().iterator(pointer);
+ while (iterator.hasNext()) {
+ TrabalAnnotation annotation = new TrabalAnnotation(iterator.next(), enableFeatures);
+ if (annotation.getBegin() >= begin) {
+ break;
+ }
+ if (annotation.getEnd() >= ruleItem.getAnnotation().getEnd()) {
+ result.add(new TrabalRuleItem(annotation));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns a list of rule items within the given bounds.
+ *
+ * @param begin
+ * @param end
+ * @param doc
+ * @param cas
+ * @return
+ */
+ private List<TrabalRuleItem> getSingleTermsWithinBounds(int begin, int end,
+ TextRulerExampleDocument doc, CAS cas) {
+ Set<TrabalRuleItem> result = new HashSet<TrabalRuleItem>();
+ Type frameType = cas.getTypeSystem().getType(ANNOTATION_TYPE_FRAME);
+ AnnotationFS pointer = cas.createAnnotation(frameType, begin, end);
+ FSIterator<AnnotationFS> iterator = cas.getAnnotationIndex().iterator(pointer);
+ while (iterator.hasNext()) {
+ TrabalAnnotation annotation = new TrabalAnnotation(iterator.next(), enableFeatures);
+ if (annotation.getBegin() > end) {
+ break;
+ }
+ if (!filterSet.contains(annotation.getType().getName())
+ && !annotation.getType().getName().equals(ANNOTATION_TYPE_BASIC)) {
+ result.add(new TrabalRuleItem(annotation));
+ }
+ }
+ return new ArrayList<TrabalRuleItem>(result);
+ }
+
+ /**
+ * Applies a set of rules to the additional data and compares it to the training data to rate
+ * them.
+ *
+ * @param rules
+ * @param documents
+ * @param additionalDocuments
+ * @param ruleSet
+ * @param isFirstHalf
+ * @return
+ */
+ public List<TrabalRule> testTrabalRulesOnDocumentSet(List<TrabalRule> rules,
+ final TextRulerExampleDocumentSet documents,
+ final TextRulerExampleDocumentSet additionalDocuments, String ruleSet) {
+
+ if (rules.isEmpty())
+ return rules;
+
+ List<TextRulerStatisticsCollector> sums = new ArrayList<TextRulerStatisticsCollector>();
+ List<TextRulerExampleDocument> goldDocs;
+ List<TextRulerExampleDocument> additionalDocs;
+ goldDocs = documents.getDocuments();
+ additionalDocs = additionalDocuments.getDocuments();
+ CAS theTestCAS = getTestCAS();
+ for (int ruleIndex = 0; ruleIndex < rules.size(); ruleIndex++) {
+ for (int i = 0; i < goldDocs.size(); i++) {
+ String ruleInfo;
+ if (rules.get(ruleIndex).getAnnotation() != null
+ && rules.get(ruleIndex).getTargetAnnotation() != null) {
+ ruleInfo = " " + rules.get(ruleIndex).getAnnotation().getType().getShortName() + "("
+ + rules.get(ruleIndex).getAnnotation().getBegin() + ","
+ + rules.get(ruleIndex).getAnnotation().getEnd() + ") -> "
+ + rules.get(ruleIndex).getTargetAnnotation().getType().getShortName() + "("
+ + rules.get(ruleIndex).getTargetAnnotation().getBegin() + ","
+ + rules.get(ruleIndex).getTargetAnnotation().getEnd() + ")";
+ } else if (rules.get(ruleIndex).getTargetAnnotation() != null) {
+ ruleInfo = " Annotate "
+ + rules.get(ruleIndex).getTargetAnnotation().getType().getShortName() + "("
+ + rules.get(ruleIndex).getTargetAnnotation().getBegin() + ","
+ + rules.get(ruleIndex).getTargetAnnotation().getEnd() + ")";
+ } else {
+ ruleInfo = " Delete " + rules.get(ruleIndex).getAnnotation().getType().getShortName()
+ + "(" + rules.get(ruleIndex).getAnnotation().getBegin() + ","
+ + rules.get(ruleIndex).getAnnotation().getEnd() + ")";
+ }
+ TextRulerExampleDocument goldDoc = goldDocs.get(i);
+ TextRulerExampleDocument additionalDoc = additionalDocs.get(i);
+ sendStatusUpdateToDelegate("Testing " + ruleSet + ruleInfo + " on document " + (i + 1)
+ + " of " + goldDocs.size() + " : rule " + (ruleIndex + 1) + " of " + rules.size(),
+ TextRulerLearnerState.ML_RUNNING, false);
+ TextRulerStatisticsCollector sumC = new TextRulerStatisticsCollector();
+ prepareTestCas(theTestCAS, goldDoc, additionalDoc);
+ testRuleOnDocument((TrabalRule) rules.get(ruleIndex), goldDoc, additionalDoc, sumC,
+ theTestCAS);
+ if (sums.size() > ruleIndex) {
+ sums.get(ruleIndex).add(sumC);
+ } else {
+ sums.add(sumC);
+ }
+ if (shouldAbort())
+ return rules;
+ }
+ }
+ for (int ruleIndex = 0; ruleIndex < rules.size(); ruleIndex++) {
+ rules.get(ruleIndex).setCoveringStatistics(sums.get(ruleIndex));
+ }
+ sums.clear();
+ return rules;
+ }
+
+ /**
+ * Resets the test CAS and adds the annotations of the additional data.
+ *
+ * @param testCas
+ * @param goldDoc
+ * @param additionalDoc
+ */
+ private void prepareTestCas(CAS testCas, TextRulerExampleDocument goldDoc,
+ TextRulerExampleDocument additionalDoc) {
+ testCas.reset();
+ CAS goldCas = goldDoc.getCAS();
+ CAS additionalCas = additionalDoc.getCAS();
+ testCas.setDocumentText(goldCas.getDocumentText());
+
+ for (AnnotationFS fs : additionalCas.getAnnotationIndex()) {
+ Type t = testCas.getTypeSystem().getType(fs.getType().getName());
+ if (t != null) {
+ AnnotationFS createAnnotation = testCas.createAnnotation(t, fs.getBegin(), fs.getEnd());
+ testCas.addFsToIndexes(createAnnotation);
+ } else {
+ TextRulerToolkit.log("Type " + fs.getType().getName() + "is unknown in test CAS");
+ }
+ }
+ }
+
+ /**
+ * Applies a rule to an element of the additional data and compares the result to the
+ * corresponding element of the training data.
+ *
+ * @param rule
+ * @param goldDoc
+ * @param additionalDoc
+ * @param c
+ * @param testCas
+ */
+ private void testRuleOnDocument(final TrabalRule rule, final TextRulerExampleDocument goldDoc,
+ final TextRulerExampleDocument additionalDoc, final TextRulerStatisticsCollector c,
+ CAS testCas) {
+ if (TextRulerToolkit.DEBUG) {
+ MemoryWatch.watch();
+ }
+ try {
+ rule.saveToRulesFile(getTempRulesFileName());
+ if (TextRulerToolkit.DEBUG) {
+ TextRulerToolkit.writeCAStoXMIFile(testCas, tempDirectory() + "testCas.xmi");
+ }
+ ae.process(testCas);
+ removeBasics(testCas);
+ if (TextRulerToolkit.DEBUG) {
+ TextRulerToolkit.writeCAStoXMIFile(testCas, tempDirectory() + "testCasProcessed.xmi");
+ }
+ if (rule.getAnnotation() != null && rule.getTargetAnnotation() != null) {
+ compareOriginalDocumentWithTestCAS(goldDoc, additionalDoc, testCas, new TextRulerTarget(
+ rule.getAnnotation().getType().getName(), this), c, false);
+ if (rule.getAnnotation().getType() != rule.getTargetAnnotation().getType()) {
+ compareOriginalDocumentWithTestCAS(goldDoc, additionalDoc, testCas, new TextRulerTarget(
+ rule.getTargetAnnotation().getType().getName(), this), c, false);
+ }
+ } else if (rule.getTargetAnnotation() != null) {
+ compareOriginalDocumentWithTestCAS(goldDoc, additionalDoc, testCas, new TextRulerTarget(
+ rule.getTargetAnnotation().getType().getName(), this), c, false);
+ } else {
+ compareOriginalDocumentWithTestCAS(goldDoc, additionalDoc, testCas, new TextRulerTarget(
+ rule.getAnnotation().getType().getName(), this), c, false);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Compares two documents to rate the results of an applied rule.
+ *
+ * @param goldDoc
+ * @param additionalDoc
+ * @param testCas
+ * @param target
+ * @param c
+ * @param collectNegativeExamples
+ */
+ private void compareOriginalDocumentWithTestCAS(TextRulerExampleDocument goldDoc,
+ TextRulerExampleDocument additionalDoc, CAS testCas, TextRulerTarget target,
+ TextRulerStatisticsCollector c, boolean collectNegativeExamples) {
+
+ List<TextRulerExample> goldPositives = goldDoc.createSlotInstancesForCAS(goldDoc.getCAS(),
+ target, true);
+ List<TextRulerExample> additionalPositives = additionalDoc.createSlotInstancesForCAS(
+ additionalDoc.getCAS(), target, true);
+ List<TextRulerExample> testPositives = goldDoc
+ .createSlotInstancesForCAS(testCas, target, false);
+
+ List<TextRulerExample> baseFP = new ArrayList<TextRulerExample>();
+ for (TextRulerExample e : additionalPositives) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(goldPositives,
+ e.getAnnotation());
+ if (example == null) {
+ baseFP.add(e);
+ }
+ }
+ List<TextRulerExample> baseFN = new ArrayList<TextRulerExample>();
+ for (TextRulerExample e : goldPositives) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(
+ additionalPositives, e.getAnnotation());
+ if (example == null) {
+ baseFN.add(e);
+ }
+ }
+
+ List<TextRulerExample> testFP = new ArrayList<TextRulerExample>();
+ for (TextRulerExample e : testPositives) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(goldPositives,
+ e.getAnnotation());
+ if (example == null) {
+ testFP.add(e);
+ }
+ }
+ List<TextRulerExample> testFN = new ArrayList<TextRulerExample>();
+ for (TextRulerExample e : goldPositives) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(testPositives,
+ e.getAnnotation());
+ if (example == null) {
+ testFN.add(e);
+ }
+ }
+
+ for (TextRulerExample e : baseFP) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(testFP,
+ e.getAnnotation());
+ if (example == null) {
+ c.addCoveredPositive(e);
+ }
+ }
+
+ for (TextRulerExample e : baseFN) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(testFN,
+ e.getAnnotation());
+ TextRulerExample coveredExample = TextRulerToolkit.exampleListContainsAnnotation(
+ goldPositives, e.getAnnotation());
+ if (example == null) {
+ c.addCoveredPositive(coveredExample);
+ }
+ }
+
+ for (TextRulerExample e : testFN) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(baseFN,
+ e.getAnnotation());
+ if (example == null) {
+ if (collectNegativeExamples) {
+ e.setPositive(false);
+ c.addCoveredNegative(e);
+ } else {
+ c.incCoveredNegatives(1);
+ }
+ }
+ }
+
+ for (TextRulerExample e : testFP) {
+ TextRulerExample example = TextRulerToolkit.exampleListContainsAnnotation(baseFP,
+ e.getAnnotation());
+ if (example == null) {
+ c.incCoveredNegatives(1);
+ }
+ }
+ }
+
+ /**
+ * Removes duplicate rules by their hash value.
+ *
+ * @param createRules
+ * @return
+ */
+ public static List<TrabalRule> removeDuplicateRules(List<TrabalRule> rules) {
+ return new ArrayList<TrabalRule>(new HashSet<TrabalRule>(rules));
+ }
+
+ /**
+ * Returns a string, containing the rules of the given list.
+ *
+ * @param rules
+ * @return
+ */
+ public static String getRuleStrings(List<TrabalRule> rules) {
+ String result = "";
+ for (TrabalRule r : rules) {
+ result += r.toString();
+ }
+ return result;
+ }
+
+ /**
+ * Tests, if the given type is a slot type.
+ *
+ * @param type
+ * @return
+ */
+ public boolean isSlotType(Type type) {
+ for (String slot : slotNames) {
+ if (slot.equals(type.getName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean collectNegativeCoveredInstancesWhenTesting() {
+ return false;
+ }
+
+ public String getResultString() {
+ return getFileHeaderString(true) + result;
+ }
+
+
+ // // @Override
+ // public AnalysisEngine getAnalysisEngine() {
+ // if (ae == null) {
+ // String descriptorFile = TextRulerToolkit.getEngineDescriptorFromTMSourceFile(new Path(
+ // preprocessorTMFile));
+ // sendStatusUpdateToDelegate("loading AE...", TextRulerLearnerState.ML_INITIALIZING, false);
+ // ae = TextRulerToolkit.loadAnalysisEngine(descriptorFile);
+ //
+ // // set filters to NO filtering so that we can add it manually with
+ // // the FILTERTYPE expression!
+ // String tempRulesFileName = getTempRulesFileName();
+ // IPath path = new Path(tempRulesFileName);
+ // ae.setConfigParameterValue(RutaEngine.MAIN_SCRIPT, path.removeFileExtension()
+ // .lastSegment());
+ // String portableString = path.removeLastSegments(1).toPortableString();
+ // ae.setConfigParameterValue(RutaEngine.SCRIPT_PATHS, new String[] { portableString });
+ // ae.setConfigParameterValue(RutaEngine.ADDITIONAL_SCRIPTS, new String[0]);
+ // ae.setConfigParameterValue(RutaEngine.RELOAD_SCRIPT, true);
+ // ae.setConfigParameterValue(RutaEngine.REMOVE_BASICS, true);
+ //
+ // try {
+ // ae.reconfigure();
+ // } catch (ResourceConfigurationException e) {
+ // TextRulerPlugin.error(e);
+ // return null;
+ // }
+ // }
+ // return ae;
+ // }
+
+ /**
+ * Return the set of additional documents and initializes it, if needed.
+ */
+ public TextRulerExampleDocumentSet getAdditionalDocuments() {
+ if (additionalDocuments == null) {
+ additionalDocuments = new TextRulerExampleDocumentSet(additionalFolderPath, casCache);
+ }
+ return additionalDocuments;
+ }
+
+ public boolean getEnableFeatures() {
+ return enableFeatures;
+ }
+
+ // @Override
+ public void setParameters(Map<String, Object> params) {
+ if (TextRulerToolkit.DEBUG)
+ saveParametersToTempFolder(params);
+ if (params.containsKey(MAX_NUMBER_OF_BASIC_RULES_KEY))
+ maxNumberOfBasicRules = (Integer) params.get(MAX_NUMBER_OF_BASIC_RULES_KEY);
+ if (params.containsKey(MAX_NUMBER_OF_RULES_KEY))
+ maxNumberOfRules = (Integer) params.get(MAX_NUMBER_OF_RULES_KEY);
+ if (params.containsKey(MAX_NUMBER_OF_ITERATIONS_KEY))
+ maxNumberOfIterations = (Integer) params.get(MAX_NUMBER_OF_ITERATIONS_KEY);
+ if (params.containsKey(ALGORITHM_ITERATIONS_KEY))
+ algorithmIterations = (Integer) params.get(ALGORITHM_ITERATIONS_KEY);
+ if (params.containsKey(MAX_ERROR_RATE_KEY))
+ maxErrorRate = (Double) params.get(MAX_ERROR_RATE_KEY);
+ if (params.containsKey(ENABLE_FEATURES_KEY))
+ enableFeatures = (Boolean) params.get(ENABLE_FEATURES_KEY);
+ }
+
+ /**
+ * Creates a file in the *.csv format, containing TraBaL errors (error type, file name, false
+ * annotation and true annotation).
+ *
+ * @throws Exception
+ */
+ public void getErrorsAsCSV(String filePath) throws Exception {
+ String result = "";
+ if (exampleDocuments == null) {
+ exampleDocuments = new TextRulerExampleDocumentSet(inputDirectory, casCache);
+ }
+ getAdditionalDocuments();
+ errors = createErrorList();
+ result += "'ErrorType';'FileName';'AnnotationType';'Annotation';'TargetAnnotationType';'TargetAnnotation';\n";
+ for (AnnotationError each : errors) {
+ if (each.getAnnotation() != null && each.getTargetAnnotation() != null) {
+ result += "'" + each.getType() + "';'"
+ + new File(each.getAnnotation().getDocument().getCasFileName()).getName() + "';'"
+ + each.getAnnotation().getType().getShortName() + "';'"
+ + each.getAnnotation().getCoveredText() + "';'"
+ + each.getTargetAnnotation().getType().getShortName() + "';'"
+ + each.getTargetAnnotation().getCoveredText() + "';\n";
+ } else if (each.getAnnotation() != null) {
+ result += "'" + each.getType() + "';'"
+ + new File(each.getAnnotation().getDocument().getCasFileName()).getName() + "';'"
+ + each.getAnnotation().getType().getShortName() + "';'"
+ + each.getAnnotation().getCoveredText() + "';'';'';\n";
+ } else {
+ result += "'" + each.getType() + "';'"
+ + new File(each.getTargetAnnotation().getDocument().getCasFileName()).getName()
+ + "';'';'';'" + each.getTargetAnnotation().getType().getShortName() + "';'"
+ + each.getTargetAnnotation().getCoveredText() + "';\n";
+ }
+ }
+ if (errors.size() > 0) {
+ File csv = new File(filePath);
+ FileWriter fstream = new FileWriter(csv);
+ BufferedWriter out = new BufferedWriter(fstream);
+ out.write(result);
+ out.close();
+ }
+ }
+
+ /**
+ * Compares trabal rules, disregarding the error rate, so basic rules with maximum number of
+ * covered positives are chosen.
+ */
+ protected Comparator<TrabalRule> basicComparator = new Comparator<TrabalRule>() {
+ public int compare(TrabalRule o1, TrabalRule o2) {
+ // coveredPositives
+ if (o1.getCoveringStatistics().getCoveredPositivesCount() > o2.getCoveringStatistics()
+ .getCoveredPositivesCount())
+ return -1;
+ if (o1.getCoveringStatistics().getCoveredPositivesCount() < o2.getCoveringStatistics()
+ .getCoveredPositivesCount())
+ return 1;
+ // coveredNegatives
+ if (o1.getCoveringStatistics().getCoveredNegativesCount() < o2.getCoveringStatistics()
+ .getCoveredNegativesCount())
+ return -1;
+ if (o1.getCoveringStatistics().getCoveredNegativesCount() > o2.getCoveringStatistics()
+ .getCoveredNegativesCount())
+ return 1;
+ // complexity of rules
+ if (o1.getConditions().size() < o2.getConditions().size())
+ return -1;
+ if (o1.getConditions().size() > o2.getConditions().size())
+ return 1;
+ // ruleString
+ return o1.getRuleString().compareTo(o2.getRuleString());
+ }
+ };
+
+ /**
+ * Compares trabal rules, starting with the error rate.
+ */
+ protected Comparator<TrabalRule> enhancedComparator = new Comparator<TrabalRule>() {
+ public int compare(TrabalRule o1, TrabalRule o2) {
+ // positives - negatives
+ if (o1.getCoveringStatistics().getCoveredPositivesCount()
+ - o1.getCoveringStatistics().getCoveredNegativesCount() > o2.getCoveringStatistics()
+ .getCoveredPositivesCount() - o2.getCoveringStatistics().getCoveredNegativesCount())
+ return -1;
+ if (o1.getCoveringStatistics().getCoveredPositivesCount()
+ - o1.getCoveringStatistics().getCoveredNegativesCount() < o2.getCoveringStatistics()
+ .getCoveredPositivesCount() - o2.getCoveringStatistics().getCoveredNegativesCount())
+ return 1;
+ // coveredPositives
+ if (o1.getCoveringStatistics().getCoveredPositivesCount() > o2.getCoveringStatistics()
+ .getCoveredPositivesCount())
+ return -1;
+ if (o1.getCoveringStatistics().getCoveredPositivesCount() < o2.getCoveringStatistics()
+ .getCoveredPositivesCount())
+ return 1;
+ // coveredNegatives
+ if (o1.getCoveringStatistics().getCoveredNegativesCount() < o2.getCoveringStatistics()
+ .getCoveredNegativesCount())
+ return -1;
+ if (o1.getCoveringStatistics().getCoveredNegativesCount() > o2.getCoveringStatistics()
+ .getCoveredNegativesCount())
+ return 1;
+ // complexity of rules
+ if (o1.getConditions().size() < o2.getConditions().size())
+ return -1;
+ if (o1.getConditions().size() > o2.getConditions().size())
+ return 1;
+ // ruleString
+ return o1.getRuleString().compareTo(o2.getRuleString());
+ }
+ };
+}