You are viewing a plain text version of this content. The canonical link for it is here.

Posted to dev@ctakes.apache.org by "Chen, Pei" <Pe...@childrens.harvard.edu> on 2014/11/10 18:11:20 UTC

Re: svn commit: r1637884 - in /ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal: ae/ eval/

Chen,
Does this need to go into this upcoming release or can it wait till the next one?

Sent from my iPhone

> On Nov 10, 2014, at 10:20 AM, "clin@apache.org" <cl...@apache.org> wrote:
> 
> Author: clin
> Date: Mon Nov 10 15:19:55 2014
> New Revision: 1637884
> 
> URL: http://svn.apache.org/r1637884
> Log:
> add annotators and update evaluation code for i2b2 data.
> add more system-generated events for candidate temporal relations.
> 
> Added:
>    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java
>    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java
> Modified:
>    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java
>    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java
>    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
>    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java
> 
> Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java
> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java?rev=1637884&view=auto
> ==============================================================================
> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java (added)
> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java Mon Nov 10 15:19:55 2014
> @@ -0,0 +1,280 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + *   http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.ctakes.temporal.ae;
> +
> +import java.io.File;
> +import java.util.ArrayList;
> +import java.util.Arrays;
> +import java.util.Collection;
> +import java.util.List;
> +import java.util.Map;
> +
> +import org.apache.ctakes.relationextractor.ae.RelationExtractorAnnotator;
> +import org.apache.ctakes.relationextractor.ae.features.PartOfSpeechFeaturesExtractor;
> +import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
> +//import org.apache.ctakes.relationextractor.ae.features.TokenFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.CheckSpecialWordRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.ConjunctionRelationFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.DependencyParseUtils;
> +import org.apache.ctakes.temporal.ae.feature.DependencyPathFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.CoordinateFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.DependingVerbsFeatureExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.EventInBetweenPropertyExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.EventOutsidePropertyExtractor;
> +import org.apache.ctakes.temporal.ae.feature.SpecialAnnotationRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TemporalPETFlatExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TokenPropertyFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.DeterminerRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.EventArgumentPropertyExtractor;
> +import org.apache.ctakes.temporal.ae.feature.EventTimeRelationFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.EventPositionRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.NumberOfEventsInTheSameSentenceExtractor;
> +import org.apache.ctakes.temporal.ae.feature.NearbyVerbTenseRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.NumberOfEventTimeBetweenCandidatesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.OverlappedHeadFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.SRLRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TimeXRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.SectionHeaderRelationExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.TemporalAttributeFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.UmlsFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.UnexpandedTokenFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.treekernel.TemporalPETExtractor;
> +import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
> +import org.apache.ctakes.typesystem.type.relation.RelationArgument;
> +import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
> +//import org.apache.ctakes.typesystem.type.syntax.ConllDependencyNode;
> +import org.apache.ctakes.typesystem.type.textsem.EventMention;
> +import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
> +import org.apache.ctakes.typesystem.type.textspan.Sentence;
> +import org.apache.uima.analysis_engine.AnalysisEngineDescription;
> +import org.apache.uima.jcas.JCas;
> +import org.apache.uima.jcas.tcas.Annotation;
> +import org.apache.uima.resource.ResourceInitializationException;
> +import org.cleartk.ml.CleartkAnnotator;
> +import org.cleartk.ml.DataWriter;
> +import org.cleartk.ml.jar.DefaultDataWriterFactory;
> +import org.cleartk.ml.jar.DirectoryDataWriterFactory;
> +import org.cleartk.ml.jar.GenericJarClassifierFactory;
> +import org.apache.uima.fit.factory.AnalysisEngineFactory;
> +import org.apache.uima.fit.util.JCasUtil;
> +
> +import com.google.common.collect.Lists;
> +
> +public class EventEventI2B2RelationAnnotator extends RelationExtractorAnnotator {
> +
> +    public static AnalysisEngineDescription createDataWriterDescription(
> +            Class<? extends DataWriter<String>> dataWriterClass,
> +                    File outputDirectory,
> +                    double probabilityOfKeepingANegativeExample) throws ResourceInitializationException {
> +        return AnalysisEngineFactory.createEngineDescription(
> +                EventEventI2B2RelationAnnotator.class,
> +                CleartkAnnotator.PARAM_IS_TRAINING,
> +                true,
> +                DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
> +                dataWriterClass,
> +                DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
> +                outputDirectory,
> +                RelationExtractorAnnotator.PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
> +                // not sure why this has to be cast; something funny going on in uimaFIT maybe?
> +                (float) probabilityOfKeepingANegativeExample);
> +    }
> +
> +    public static AnalysisEngineDescription createAnnotatorDescription(String modelPath)
> +            throws ResourceInitializationException {
> +        return AnalysisEngineFactory.createEngineDescription(
> +                EventEventI2B2RelationAnnotator.class,
> +                CleartkAnnotator.PARAM_IS_TRAINING,
> +                false,
> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
> +                modelPath);
> +    }
> +
> +    /**
> +     * @deprecated use String path instead of File.
> +     * ClearTK will automatically Resolve the String to an InputStream.
> +     * This will allow resources to be read within from a jar as well as File.  
> +     */      
> +    @SuppressWarnings("dep-ann")
> +    public static AnalysisEngineDescription createAnnotatorDescription(File modelDirectory)
> +            throws ResourceInitializationException {
> +        return AnalysisEngineFactory.createEngineDescription(
> +                EventEventI2B2RelationAnnotator.class,
> +                CleartkAnnotator.PARAM_IS_TRAINING,
> +                false,
> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
> +                new File(modelDirectory, "model.jar"));
> +    }
> +
> +    @Override
> +    protected List<RelationFeaturesExtractor> getFeatureExtractors() {
> +        return Lists.newArrayList(
> +                //                new TokenFeaturesExtractor()
> +                new UnexpandedTokenFeaturesExtractor() //use unexpanded version for i2b2 data
> +                , new PartOfSpeechFeaturesExtractor()
> +                //                , new TemporalPETExtractor()
> +                , new EventArgumentPropertyExtractor()
> +                , new NumberOfEventTimeBetweenCandidatesExtractor()
> +                , new SectionHeaderRelationExtractor()
> +                , new NearbyVerbTenseRelationExtractor()
> +                , new CheckSpecialWordRelationExtractor()
> +                , new UmlsFeatureExtractor()
> +                , new DependencyPathFeaturesExtractor()
> +                , new CoordinateFeaturesExtractor()
> +                , new OverlappedHeadFeaturesExtractor()
> +                , new SRLRelationFeaturesExtractor()
> +                , new NumberOfEventsInTheSameSentenceExtractor()
> +                , new EventPositionRelationFeaturesExtractor() //not helpful
> +                , new TimeXRelationFeaturesExtractor() //not helpful
> +                , new ConjunctionRelationFeaturesExtractor()
> +                , new DeterminerRelationFeaturesExtractor()
> +                , new EventTimeRelationFeatureExtractor()
> +                , new TokenPropertyFeaturesExtractor()
> +                , new DependingVerbsFeatureExtractor()
> +                , new SpecialAnnotationRelationExtractor() //not helpful
> +                , new TemporalPETFlatExtractor()
> +                //                , new EventInBetweenPropertyExtractor()
> +                //                , new EventOutsidePropertyExtractor()
> +                );
> +    }
> +
> +    @Override
> +    protected Class<? extends Annotation> getCoveringClass() {
> +        return Sentence.class;
> +    }
> +
> +    @Override
> +    protected List<IdentifiedAnnotationPair> getCandidateRelationArgumentPairs(
> +            JCas jCas, Annotation sentence) {
> +
> +        Map<EventMention, Collection<EventMention>> coveringMap =
> +                JCasUtil.indexCovering(jCas, EventMention.class, EventMention.class);
> +
> +        List<IdentifiedAnnotationPair> pairs = Lists.newArrayList();
> +        List<EventMention> events = new ArrayList<>(JCasUtil.selectCovered(jCas, EventMention.class, sentence));
> +        //filter events:
> +        List<EventMention> realEvents = Lists.newArrayList();
> +        for( EventMention event : events){
> +            if(event.getClass().equals(EventMention.class)){
> +                realEvents.add(event);
> +            }
> +        }
> +        events = realEvents;
> +
> +        int eventNum = events.size();
> +
> +        for (int i = 0; i < eventNum-1; i++){
> +            for(int j = i+1; j < eventNum; j++){
> +                EventMention eventA = events.get(j);
> +                EventMention eventB = events.get(i);
> +
> +                if(this.isTraining()){
> +                    //pairing covering system events:
> +                    for (EventMention event1 : coveringMap.get(eventA)){
> +                        for(EventMention event2 : coveringMap.get(eventB)){
> +                            pairs.add(new IdentifiedAnnotationPair(event1, event2));                            
> +                        }
> +                        pairs.add(new IdentifiedAnnotationPair(event1, eventB));
> +                    }
> +                    for(EventMention event2 : coveringMap.get(eventB)){
> +                        pairs.add(new IdentifiedAnnotationPair(eventA, event2));                            
> +                    }
> +                    //pairing covered system events:
> +                    for(EventMention event1 : JCasUtil.selectCovered(jCas, EventMention.class, eventA)){
> +                        for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
> +                            pairs.add(new IdentifiedAnnotationPair(event1, event2));
> +                        }
> +                        pairs.add(new IdentifiedAnnotationPair(event1, eventB));
> +                    }
> +                    for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
> +                        pairs.add(new IdentifiedAnnotationPair(eventA, event2));
> +                    }
> +                }
> +                pairs.add(new IdentifiedAnnotationPair(eventA, eventB));
> +
> +            }
> +        }
> +
> +
> +        //        if(eventNum >= 2){
> +        //            for ( int i = 0; i< eventNum -1 ; i ++){
> +        //                EventMention evI = events.get(i);
> +        //                for(int j = i+1; j< eventNum; j++){
> +        //                    EventMention evJ = events.get(j);
> +        //                    if(j-i==1 || j-i==eventNum-1){//if two events are consecutive, or major events
> +        //                        pairs.add(new IdentifiedAnnotationPair(evJ, evI));
> +        //                    }else if(ifDependent(jCas, evI, evJ)){//if the event pairs are dependent// eventNum < 7 && 
> +        //                        pairs.add(new IdentifiedAnnotationPair(evJ, evI));
> +        //                    }else{// if the 
> +        //                        continue;
> +        //                    }
> +        //                }
> +        //            }
> +        //        }
> +
> +        return pairs;
> +    }
> +
> +    @Override
> +    protected void createRelation(JCas jCas, IdentifiedAnnotation arg1,
> +            IdentifiedAnnotation arg2, String predictedCategory) {
> +        RelationArgument relArg1 = new RelationArgument(jCas);
> +        relArg1.setArgument(arg1);
> +        relArg1.setRole("Arg1");
> +        relArg1.addToIndexes();
> +        RelationArgument relArg2 = new RelationArgument(jCas);
> +        relArg2.setArgument(arg2);
> +        relArg2.setRole("Arg2");
> +        relArg2.addToIndexes();
> +        TemporalTextRelation relation = new TemporalTextRelation(jCas);
> +        relation.setArg1(relArg1);
> +        relation.setArg2(relArg2);
> +        relation.setCategory(predictedCategory);
> +        relation.addToIndexes();
> +    }
> +
> +    @Override
> +    protected String getRelationCategory(
> +            Map<List<Annotation>, BinaryTextRelation> relationLookup,
> +            IdentifiedAnnotation arg1,
> +            IdentifiedAnnotation arg2) {
> +        BinaryTextRelation relation = relationLookup.get(Arrays.asList(arg1, arg2));
> +        String category = null;
> +        if (relation != null && relation instanceof TemporalTextRelation) {
> +            category = relation.getCategory();
> +        } else {
> +            relation = relationLookup.get(Arrays.asList(arg2, arg1));
> +            if (relation != null && relation instanceof TemporalTextRelation) {
> +                if(relation.getCategory().equals("OVERLAP")){
> +                    category = relation.getCategory();
> +                    //                }else if (relation.getCategory().equals("BEFORE")){
> +                    //                    category = "AFTER";
> +                    //                }else if (relation.getCategory().equals("AFTER")){
> +                    //                    category = "BEFORE";
> +                    //                }
> +                }else{
> +                    category = relation.getCategory() + "-1";
> +                }
> +            }
> +        }
> +        if (category == null && coin.nextDouble() <= this.probabilityOfKeepingANegativeExample) {
> +            category = NO_RELATION_CATEGORY;
> +        }
> +        return category;
> +    }
> +}
> 
> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java
> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java?rev=1637884&r1=1637883&r2=1637884&view=diff
> ==============================================================================
> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java (original)
> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java Mon Nov 10 15:19:55 2014
> @@ -198,15 +198,25 @@ public class EventEventRelationAnnotator
>                }
>                if(eventAValid && eventBValid){
>                    if(this.isTraining()){
> +                        //pairing covering system events:
>                        for (EventMention event1 : coveringMap.get(eventA)){
>                            for(EventMention event2 : coveringMap.get(eventB)){
>                                pairs.add(new IdentifiedAnnotationPair(event1, event2));
>                            }
> +                            pairs.add(new IdentifiedAnnotationPair(event1, eventB));
>                        }
> +                        for(EventMention event2 : coveringMap.get(eventB)){
> +                            pairs.add(new IdentifiedAnnotationPair(eventA, event2));
> +                        }
> +                        //pairing covered system events:
>                        for(EventMention event1 : JCasUtil.selectCovered(jCas, EventMention.class, eventA)){
>                            for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
>                                pairs.add(new IdentifiedAnnotationPair(event1, event2));
>                            }
> +                            pairs.add(new IdentifiedAnnotationPair(event1, eventB));
> +                        }
> +                        for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
> +                            pairs.add(new IdentifiedAnnotationPair(eventA, event2));
>                        }
>                    }
>                    pairs.add(new IdentifiedAnnotationPair(eventA, eventB));
> 
> Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java
> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java?rev=1637884&view=auto
> ==============================================================================
> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java (added)
> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java Mon Nov 10 15:19:55 2014
> @@ -0,0 +1,253 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + *   http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.ctakes.temporal.ae;
> +
> +import java.io.File;
> +import java.util.Arrays;
> +import java.util.Collection;
> +import java.util.List;
> +import java.util.Map;
> +
> +import org.apache.ctakes.relationextractor.ae.RelationExtractorAnnotator;
> +import org.apache.ctakes.relationextractor.ae.features.PartOfSpeechFeaturesExtractor;
> +import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.CheckSpecialWordRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.ConjunctionRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.DependencyFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.DependencyPathFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.EventArgumentPropertyExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.EventIndexOfSameSentenceRelationFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.EventPositionRelationFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.EventTimeRelationFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.NearbyVerbTenseRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.NearestFlagFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.NumberOfEventTimeBetweenCandidatesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.OverlappedHeadFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.SRLRelationFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.SectionHeaderRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TemporalAttributeFeatureExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TemporalPETFlatExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.TimeWordTypeRelationExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TimeXPropertyRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.TimeXRelationFeaturesExtractor;
> +import org.apache.ctakes.temporal.ae.feature.UnexpandedTokenFeaturesExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.TemporalAttributeFeatureExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.treekernel.EventTimeFlatTreeFeatureExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.treekernel.EventVerbRelationTreeExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.treekernel.TemporalPETExtractor;
> +//import org.apache.ctakes.temporal.ae.feature.treekernel.TemporalPathExtractor;
> +import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
> +import org.apache.ctakes.typesystem.type.relation.RelationArgument;
> +import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
> +import org.apache.ctakes.typesystem.type.textsem.EventMention;
> +import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
> +import org.apache.ctakes.typesystem.type.textsem.TimeMention;
> +import org.apache.ctakes.typesystem.type.textspan.Sentence;
> +import org.apache.uima.analysis_engine.AnalysisEngineDescription;
> +import org.apache.uima.jcas.JCas;
> +import org.apache.uima.jcas.tcas.Annotation;
> +import org.apache.uima.resource.ResourceInitializationException;
> +import org.cleartk.ml.CleartkAnnotator;
> +import org.cleartk.ml.DataWriter;
> +import org.cleartk.ml.jar.DefaultDataWriterFactory;
> +import org.cleartk.ml.jar.DirectoryDataWriterFactory;
> +import org.cleartk.ml.jar.GenericJarClassifierFactory;
> +import org.apache.uima.fit.factory.AnalysisEngineFactory;
> +import org.apache.uima.fit.util.JCasUtil;
> +
> +import com.google.common.collect.Lists;
> +
> +public class EventTimeI2B2RelationAnnotator extends RelationExtractorAnnotator {
> +
> +    public static AnalysisEngineDescription createDataWriterDescription(
> +            Class<? extends DataWriter<String>> dataWriterClass,
> +                    File outputDirectory,
> +                    double probabilityOfKeepingANegativeExample) throws ResourceInitializationException {
> +        return AnalysisEngineFactory.createEngineDescription(
> +                EventTimeI2B2RelationAnnotator.class,
> +                CleartkAnnotator.PARAM_IS_TRAINING,
> +                true,
> +                DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
> +                dataWriterClass,
> +                DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
> +                outputDirectory,
> +                RelationExtractorAnnotator.PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
> +                // not sure why this has to be cast; something funny going on in uimaFIT maybe?
> +                (float) probabilityOfKeepingANegativeExample);
> +    }
> +
> +    public static AnalysisEngineDescription createEngineDescription(String modelPath)
> +            throws ResourceInitializationException {
> +        return AnalysisEngineFactory.createEngineDescription(
> +                EventTimeI2B2RelationAnnotator.class,
> +                CleartkAnnotator.PARAM_IS_TRAINING,
> +                false,
> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
> +                modelPath);
> +    }
> +    /**
> +     * @deprecated use String path instead of File.
> +     * ClearTK will automatically Resolve the String to an InputStream.
> +     * This will allow resources to be read within from a jar as well as File.  
> +     */     
> +    @Deprecated
> +    public static AnalysisEngineDescription createEngineDescription(File modelDirectory)
> +            throws ResourceInitializationException {
> +        return AnalysisEngineFactory.createEngineDescription(
> +                EventTimeI2B2RelationAnnotator.class,
> +                CleartkAnnotator.PARAM_IS_TRAINING,
> +                false,
> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
> +                new File(modelDirectory, "model.jar"));
> +    }
> +
> +    @Override
> +    protected List<RelationFeaturesExtractor> getFeatureExtractors() {
> +        return Lists.newArrayList(
> +//                new TokenFeaturesExtractor()
> +                new UnexpandedTokenFeaturesExtractor() //use unexpanded version for i2b2 data
> +                , new PartOfSpeechFeaturesExtractor()
> +                , new TemporalAttributeFeatureExtractor()
> +                //                , new EventTimeFlatTreeFeatureExtractor()
> +                //                , new TemporalPETExtractor()
> +                //, new TemporalPathExtractor()
> +                //                , new EventVerbRelationTreeExtractor()
> +                , new NumberOfEventTimeBetweenCandidatesExtractor()
> +                //                , new SectionHeaderRelationExtractor()
> +                , new NearbyVerbTenseRelationExtractor()
> +                , new CheckSpecialWordRelationExtractor()
> +                , new NearestFlagFeatureExtractor()
> +                , new DependencyPathFeaturesExtractor()
> +                , new DependencyFeatureExtractor()
> +                //                , new SRLRelationFeaturesExtractor()// tried, but not helpful
> +                , new EventArgumentPropertyExtractor()
> +                , new OverlappedHeadFeaturesExtractor()
> +                //                , new EventTimeRelationFeatureExtractor()
> +                , new ConjunctionRelationFeaturesExtractor()
> +                //                , new EventPositionRelationFeaturesExtractor() //tried, but not helpful
> +                , new TimeXRelationFeaturesExtractor()
> +                , new TemporalPETFlatExtractor()
> +                , new TimeXPropertyRelationFeaturesExtractor()
> +                //                , new TimeWordTypeRelationExtractor() //tried, but not helpful
> +                //                , new EventIndexOfSameSentenceRelationFeaturesExtractor() //tried, but not helpful
> +                );
> +    }
> +
> +    @Override
> +    protected Class<? extends Annotation> getCoveringClass() {
> +        return Sentence.class;
> +    }
> +
> +    @Override
> +    public List<IdentifiedAnnotationPair> getCandidateRelationArgumentPairs(
> +            JCas jCas,
> +            Annotation sentence) {
> +        Map<EventMention, Collection<EventMention>> coveringMap =
> +                JCasUtil.indexCovering(jCas, EventMention.class, EventMention.class);
> +
> +        List<IdentifiedAnnotationPair> pairs = Lists.newArrayList();
> +        for (EventMention event : JCasUtil.selectCovered(jCas, EventMention.class, sentence)) {
> +            boolean eventValid = false;
> +            if (event.getClass().equals(EventMention.class)) {//event is a gold event
> +                eventValid = true;
> +            }
> +
> +            if(eventValid){
> +                // ignore subclasses like Procedure and Disease/Disorder
> +                if(this.isTraining()){//if training mode, train on both gold event and span-overlapping system events
> +                    for (TimeMention time : JCasUtil.selectCovered(jCas, TimeMention.class, sentence)) {
> +                        Collection<EventMention> eventList = coveringMap.get(event);
> +                        for(EventMention covEvent : eventList){
> +                            pairs.add(new IdentifiedAnnotationPair(covEvent, time));
> +                        }
> +                        pairs.add(new IdentifiedAnnotationPair(event, time));
> +                        for(EventMention covedEvent : JCasUtil.selectCovered(jCas, EventMention.class, event)){//select covered events
> +                            pairs.add(new IdentifiedAnnotationPair(covedEvent, time));
> +                        }
> +                    }
> +                }else{//if testing mode, only test on system generated events
> +                    for (TimeMention time : JCasUtil.selectCovered(jCas, TimeMention.class, sentence)) {
> +                        pairs.add(new IdentifiedAnnotationPair(event, time));
> +                    }
> +                }
> +            }
> +        }
> +
> +        return pairs;
> +    }
> +
> +    @Override
> +    protected void createRelation(JCas jCas, IdentifiedAnnotation arg1,
> +            IdentifiedAnnotation arg2, String predictedCategory) {
> +        RelationArgument relArg1 = new RelationArgument(jCas);
> +        relArg1.setArgument(arg1);
> +        relArg1.setRole("Arg1");
> +        relArg1.addToIndexes();
> +        RelationArgument relArg2 = new RelationArgument(jCas);
> +        relArg2.setArgument(arg2);
> +        relArg2.setRole("Arg2");
> +        relArg2.addToIndexes();
> +        TemporalTextRelation relation = new TemporalTextRelation(jCas);
> +        relation.setArg1(relArg1);
> +        relation.setArg2(relArg2);
> +        relation.setCategory(predictedCategory);
> +        relation.addToIndexes();
> +    }
> +
> +
> +    @Override
> +    protected String getRelationCategory(
> +            Map<List<Annotation>, BinaryTextRelation> relationLookup,
> +            IdentifiedAnnotation arg1,
> +            IdentifiedAnnotation arg2) {
> +        BinaryTextRelation relation = relationLookup.get(Arrays.asList(arg1, arg2));
> +        String category = null;
> +        if (relation != null) {
> +            category = relation.getCategory();
> +        } else {
> +            relation = relationLookup.get(Arrays.asList(arg2, arg1));
> +            if (relation != null) {
> +                if(relation.getCategory().equals("OVERLAP")){
> +                    category = relation.getCategory();
> +                    //                }else if (relation.getCategory().equals("BEFORE")){
> +                    //                    category = "AFTER";
> +                    //                }else if (relation.getCategory().equals("AFTER")){
> +                    //                    category = "BEFORE";
> +                    //                }
> +                }else{
> +                    category = relation.getCategory() + "-1";
> +                }
> +            }
> +        }
> +
> +        if(category!=null){
> +            if(!((EventMention)arg1).getClass().equals(EventMention.class)){
> +                System.out.println("find system-event relations: "+ arg1.getCoveredText() + " -"+category+"- " + arg2.getCoveredText());
> +            }else{
> +                System.out.println("find gold-event relations: "+ arg1.getCoveredText() + " -"+category+"- " + arg2.getCoveredText());
> +            }
> +        }
> +
> +        if (category == null && coin.nextDouble() <= this.probabilityOfKeepingANegativeExample) {
> +            category = NO_RELATION_CATEGORY;
> +        }
> +
> +        return category;
> +    }
> +}
> 
> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java
> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java?rev=1637884&r1=1637883&r2=1637884&view=diff
> ==============================================================================
> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java (original)
> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java Mon Nov 10 15:19:55 2014
> @@ -329,7 +329,7 @@ EvaluationOfTemporalRelations_ImplBase{
>        }
> 
>        //    HideOutput hider = new HideOutput();
> -        JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"),"-c", optArray[1]);//"-w1","0.09","-w2","4","-w3","9","-w5","2","-w6","16","-w7","10","-w8","6", "-w9","45","-w10","30","-c", optArray[1]);//"-c", "0.05");//optArray);
> +        JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"),"-w1","0.08","-w3","3","-w4","17","-w5","20","-w6","16","-w7","10","-w8","6", "-w9","45","-w10","30","-c", optArray[1]);//"-c", "0.05");//optArray);
>        //        JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"), "-h","0","-c", "1000");
>        //    hider.restoreOutput();
>        //    hider.close();
> @@ -529,6 +529,20 @@ EvaluationOfTemporalRelations_ImplBase{
>                        for(EventMention covEventB : coveringMap.get(event2)){
>                            createRelation(relationView, covEventA, covEventB, relation.getCategory());
>                        }
> +                        createRelation(relationView, covEventA, event2, relation.getCategory());
> +                    }
> +                    for(EventMention covEventB : coveringMap.get(event2)){
> +                        createRelation(relationView, event1, covEventB, relation.getCategory());
> +                    }
> +                    //get covered system events:
> +                    for(EventMention covedEventA : JCasUtil.selectCovered(jCas, EventMention.class, event1)){//select covered events
> +                        for(EventMention covedEventB : JCasUtil.selectCovered(jCas, EventMention.class, event2)){
> +                            createRelation(relationView, covedEventA, covedEventB, relation.getCategory());
> +                        }
> +                        createRelation(relationView, covedEventA, event2, relation.getCategory());
> +                    }
> +                    for(EventMention covedEventB : JCasUtil.selectCovered(jCas, EventMention.class, event2)){
> +                        createRelation(relationView, event1, covedEventB, relation.getCategory());
>                    }
>                }
>            }
> 
> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java?rev=1637884&r1=1637883&r2=1637884&view=diff
> ==============================================================================
> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java (original)
> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java Mon Nov 10 15:19:55 2014
> @@ -34,6 +34,7 @@ import java.util.List;
> import java.util.Map;
> import java.util.Set;
> 
> +import org.apache.ctakes.relationextractor.ae.RelationExtractorAnnotator.IdentifiedAnnotationPair;
> import org.apache.ctakes.relationextractor.eval.RelationExtractorEvaluation.HashableArguments;
> import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
> import org.apache.ctakes.temporal.ae.EventTimeSelfRelationAnnotator;
> @@ -693,6 +694,9 @@ EvaluationOfTemporalRelations_ImplBase{
>                            createRelation(relationView, covEvent, arg2, relation.getCategory());
>                        }
>                    }
> +                    for(EventMention covedEvent : JCasUtil.selectCovered(jCas, EventMention.class, event)){//select covered events
> +                        createRelation(relationView, covedEvent, arg2, relation.getCategory());
> +                    }
>                }else if(arg2 instanceof EventMention && arg1 instanceof TimeMention){
>                    event = (EventMention) arg2;
>                    Collection<EventMention> eventList = coveringMap.get(event);
> @@ -701,6 +705,9 @@ EvaluationOfTemporalRelations_ImplBase{
>                            createRelation(relationView, arg1, covEvent, relation.getCategory());
>                        }
>                    }
> +                    for(EventMention covedEvent : JCasUtil.selectCovered(jCas, EventMention.class, event)){//select covered events
> +                        createRelation(relationView, arg1, covedEvent, relation.getCategory());
> +                    }
>                }
>            }
> 
> 
> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java
> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java?rev=1637884&r1=1637883&r2=1637884&view=diff
> ==============================================================================
> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java (original)
> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java Mon Nov 10 15:19:55 2014
> @@ -38,14 +38,15 @@ import org.apache.ctakes.core.ae.CDASegm
> import org.apache.ctakes.relationextractor.eval.RelationExtractorEvaluation.HashableArguments;
> import org.apache.ctakes.temporal.ae.ConsecutiveSentencesEventEventRelationAnnotator;
> import org.apache.ctakes.temporal.ae.ConsecutiveSentencesEventTimeRelationAnnotator;
> -import org.apache.ctakes.temporal.ae.EventTimeSelfRelationAnnotator;
> import org.apache.ctakes.temporal.ae.TemporalRelationRuleAnnotator;
> import org.apache.ctakes.temporal.ae.DocTimeRelAnnotator;
> import org.apache.ctakes.temporal.ae.EventAdmissionTimeAnnotator;
> import org.apache.ctakes.temporal.ae.EventDischargeTimeAnnotator;
> -import org.apache.ctakes.temporal.ae.EventEventRelationAnnotator;
> -import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
> +import org.apache.ctakes.temporal.ae.EventEventI2B2RelationAnnotator;
> +import org.apache.ctakes.temporal.ae.EventTimeI2B2RelationAnnotator;
> import org.apache.ctakes.temporal.ae.baselines.RecallBaselineEventTimeRelationAnnotator;
> +import org.apache.ctakes.temporal.eval.EvaluationOfEventEventThymeRelations.AddEEPotentialRelations;
> +import org.apache.ctakes.temporal.eval.EvaluationOfEventTimeRelations.AddPotentialRelations;
> import org.apache.ctakes.temporal.eval.EvaluationOfEventTimeRelations.ParameterSettings;
> import org.apache.ctakes.temporal.eval.EvaluationOfTemporalRelations_ImplBase.RemoveNonContainsRelations.RemoveGoldAttributes;
> import org.apache.ctakes.temporal.utils.AnnotationIdCollection;
> @@ -66,7 +67,6 @@ import org.apache.uima.fit.component.JCa
> import org.apache.uima.fit.descriptor.ConfigurationParameter;
> import org.apache.uima.fit.factory.AggregateBuilder;
> import org.apache.uima.fit.factory.AnalysisEngineFactory;
> -import org.apache.uima.fit.pipeline.JCasIterator;
> import org.apache.uima.fit.pipeline.SimplePipeline;
> import org.apache.uima.fit.util.JCasUtil;
> import org.apache.uima.jcas.JCas;
> @@ -75,7 +75,7 @@ import org.apache.uima.resource.Resource
> import org.apache.uima.util.FileUtils;
> import org.cleartk.eval.AnnotationStatistics;
> import org.cleartk.ml.jar.JarClassifierBuilder;
> -//import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter;
> +import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter;
> import org.cleartk.ml.liblinear.LibLinearStringOutcomeDataWriter;
> //import org.cleartk.classifier.tksvmlight.TKSVMlightStringOutcomeDataWriter;
> import org.cleartk.ml.tksvmlight.model.CompositeKernel.ComboOperator;
> @@ -292,13 +292,18 @@ EvaluationOfTemporalRelations_ImplBase{
>        //                CAS.NAME_DEFAULT_SOFA,
>        //                GOLD_VIEW_NAME);
>        //        aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveNonContainsRelations.class));
> -        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
> +//        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
> 
>        //        aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveEventEventRelations.class));
>        //test rules:
>        //        aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(TemporalRelationRuleAnnotator.class));
> +        
> +        //add unlabeled nearby system events as potential event-time links: 
> +        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddPotentialRelations.class));
> +        //add unlabeled nearby system events as potential event-event links: 
> +        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddEEPotentialRelations.class));
> 
> -        aggregateBuilder.add(EventTimeSelfRelationAnnotator.createDataWriterDescription(
> +        aggregateBuilder.add(EventTimeI2B2RelationAnnotator.createDataWriterDescription(
>                //                LibSvmStringOutcomeDataWriter.class,
>                LibLinearStringOutcomeDataWriter.class,
>                // TKSVMlightStringOutcomeDataWriter.class,
> @@ -306,27 +311,27 @@ EvaluationOfTemporalRelations_ImplBase{
>                //        SVMlightStringOutcomeDataWriter.class,        
>                new File(directory,EVENT_TIME),
>                params.probabilityOfKeepingANegativeExample));
> -        aggregateBuilder.add(EventEventRelationAnnotator.createDataWriterDescription(
> +        aggregateBuilder.add(EventEventI2B2RelationAnnotator.createDataWriterDescription(
>                LibLinearStringOutcomeDataWriter.class,//TKSVMlightStringOutcomeDataWriter.class,//
>                //                LIBLINEARStringOutcomeDataWriter.class,
>                new File(directory,EVENT_EVENT), 
>                params.probabilityOfKeepingANegativeExample));
>        aggregateBuilder.add(EventDischargeTimeAnnotator.createDataWriterDescription(
> -                //                LibSvmStringOutcomeDataWriter.class,
> -                LibLinearStringOutcomeDataWriter.class,
> +                LibSvmStringOutcomeDataWriter.class,
> +//                LibLinearStringOutcomeDataWriter.class,
>                new File(directory,EVENT_DISCHARGE)));
>        aggregateBuilder.add(EventAdmissionTimeAnnotator.createDataWriterDescription(
> -                //                LibSvmStringOutcomeDataWriter.class,
> -                LibLinearStringOutcomeDataWriter.class,
> +                LibSvmStringOutcomeDataWriter.class,
> +//                LibLinearStringOutcomeDataWriter.class,
>                new File(directory,EVENT_ADMISSION)));
>        aggregateBuilder.add(ConsecutiveSentencesEventEventRelationAnnotator.createDataWriterDescription(
> -                //                LibSvmStringOutcomeDataWriter.class,
> -                LibLinearStringOutcomeDataWriter.class,
> +                LibSvmStringOutcomeDataWriter.class,
> +//                LibLinearStringOutcomeDataWriter.class,
>                new File(directory,TEMP_CROSSSENT), 
>                params.probabilityOfKeepingANegativeExample));
>        aggregateBuilder.add(ConsecutiveSentencesEventTimeRelationAnnotator.createDataWriterDescription(
> -                //                LibSvmStringOutcomeDataWriter.class,
> -                LibLinearStringOutcomeDataWriter.class,
> +                LibSvmStringOutcomeDataWriter.class,
> +//                LibLinearStringOutcomeDataWriter.class,
>                new File(directory,TEMPET_CROSSSENT), 
>                params.probabilityOfKeepingANegativeExample));
>        SimplePipeline.runPipeline(collectionReader, aggregateBuilder.createAggregate());
> @@ -355,14 +360,14 @@ EvaluationOfTemporalRelations_ImplBase{
>        }
> 
>        //    HideOutput hider = new HideOutput();
> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_TIME), "-c", "0.001");//"-h","0","-c", "1000");//optArray);//"-c", "0.05");//
> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_EVENT), "-c", "0.001");
> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_DISCHARGE), "-c", "0.001");
> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_ADMISSION), "-c", "0.001");
> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_TIME), "-c", "0.0002", "-w2","0.2","-w3","7","-w4","7","-w5","13");//"-h","0","-c", "1000");//optArray);//"-c", "0.05");//
> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_EVENT), "-c", "0.0002","-w1","0.07","-w2","14","-w3","5","-w4","10","-w6","20");
> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_DISCHARGE), "-h","0","-c", "1000");
> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_ADMISSION), "-h","0","-c", "1000");
>        //        JarClassifierBuilder.trainAndPackage(new File(directory,TIME_ADMISSION), "-h","0","-c", "1000");
>        //        JarClassifierBuilder.trainAndPackage(new File(directory,TIME_DISCHARGE), "-h","0","-c", "1000");
> -        JarClassifierBuilder.trainAndPackage(new File(directory,TEMP_CROSSSENT), "-c", "0.001");
> -        JarClassifierBuilder.trainAndPackage(new File(directory,TEMPET_CROSSSENT), "-c", "0.001");
> +        JarClassifierBuilder.trainAndPackage(new File(directory,TEMP_CROSSSENT), "-h","0","-c", "1000");
> +        JarClassifierBuilder.trainAndPackage(new File(directory,TEMPET_CROSSSENT), "-h","0","-c", "1000");
>        //    hider.restoreOutput();
>        //    hider.close();
>    }
> @@ -419,8 +424,8 @@ EvaluationOfTemporalRelations_ImplBase{
>        //        aggregateBuilder.add(TimexDischargeTimeAnnotator.createAnnotatorDescription(new File(directory,TIME_DISCHARGE)));
> 
>        aggregateBuilder.add(this.baseline ? RecallBaselineEventTimeRelationAnnotator.createAnnotatorDescription(directory) :
> -            EventTimeSelfRelationAnnotator.createEngineDescription(new File(directory,EVENT_TIME)));
> -        aggregateBuilder.add(EventEventRelationAnnotator.createAnnotatorDescription(new File(directory,EVENT_EVENT)));
> +            EventTimeI2B2RelationAnnotator.createEngineDescription(new File(directory,EVENT_TIME)));
> +        aggregateBuilder.add(EventEventI2B2RelationAnnotator.createAnnotatorDescription(new File(directory,EVENT_EVENT)));
>        aggregateBuilder.add(ConsecutiveSentencesEventEventRelationAnnotator.createAnnotatorDescription(new File(directory,TEMP_CROSSSENT)));
>        aggregateBuilder.add(ConsecutiveSentencesEventTimeRelationAnnotator.createAnnotatorDescription(new File(directory,TEMPET_CROSSSENT)));
>        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(TemporalRelationRuleAnnotator.class));
> 
>

Re: svn commit: r1637884 - in /ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal: ae/ eval/

Posted by "Lin, Chen" <Ch...@childrens.harvard.edu>.

It can wait if the current models work.

Sent from my iPhone

> On Nov 10, 2014, at 12:18 PM, "Chen, Pei" <Pe...@childrens.harvard.edu> wrote:
> 
> Chen,
> Does this need to go into this upcoming release or can it wait till the next one?
> 
> Sent from my iPhone
> 
>> On Nov 10, 2014, at 10:20 AM, "clin@apache.org" <cl...@apache.org> wrote:
>> 
>> Author: clin
>> Date: Mon Nov 10 15:19:55 2014
>> New Revision: 1637884
>> 
>> URL: http://svn.apache.org/r1637884
>> Log:
>> add annotators and update evaluation code for i2b2 data.
>> add more system-generated events for candidate temporal relations.
>> 
>> Added:
>>   ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java
>>   ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java
>> Modified:
>>   ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java
>>   ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java
>>   ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
>>   ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java
>> 
>> Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java
>> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java?rev=1637884&view=auto
>> ==============================================================================
>> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java (added)
>> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventI2B2RelationAnnotator.java Mon Nov 10 15:19:55 2014
>> @@ -0,0 +1,280 @@
>> +/**
>> + * Licensed to the Apache Software Foundation (ASF) under one
>> + * or more contributor license agreements.  See the NOTICE file
>> + * distributed with this work for additional information
>> + * regarding copyright ownership.  The ASF licenses this file
>> + * to you under the Apache License, Version 2.0 (the
>> + * "License"); you may not use this file except in compliance
>> + * with the License.  You may obtain a copy of the License at
>> + *
>> + *   http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing,
>> + * software distributed under the License is distributed on an
>> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
>> + * KIND, either express or implied.  See the License for the
>> + * specific language governing permissions and limitations
>> + * under the License.
>> + */
>> +package org.apache.ctakes.temporal.ae;
>> +
>> +import java.io.File;
>> +import java.util.ArrayList;
>> +import java.util.Arrays;
>> +import java.util.Collection;
>> +import java.util.List;
>> +import java.util.Map;
>> +
>> +import org.apache.ctakes.relationextractor.ae.RelationExtractorAnnotator;
>> +import org.apache.ctakes.relationextractor.ae.features.PartOfSpeechFeaturesExtractor;
>> +import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
>> +//import org.apache.ctakes.relationextractor.ae.features.TokenFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.CheckSpecialWordRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.ConjunctionRelationFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.DependencyParseUtils;
>> +import org.apache.ctakes.temporal.ae.feature.DependencyPathFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.CoordinateFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.DependingVerbsFeatureExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.EventInBetweenPropertyExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.EventOutsidePropertyExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.SpecialAnnotationRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TemporalPETFlatExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TokenPropertyFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.DeterminerRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.EventArgumentPropertyExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.EventTimeRelationFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.EventPositionRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.NumberOfEventsInTheSameSentenceExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.NearbyVerbTenseRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.NumberOfEventTimeBetweenCandidatesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.OverlappedHeadFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.SRLRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TimeXRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.SectionHeaderRelationExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.TemporalAttributeFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.UmlsFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.UnexpandedTokenFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.treekernel.TemporalPETExtractor;
>> +import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
>> +import org.apache.ctakes.typesystem.type.relation.RelationArgument;
>> +import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
>> +//import org.apache.ctakes.typesystem.type.syntax.ConllDependencyNode;
>> +import org.apache.ctakes.typesystem.type.textsem.EventMention;
>> +import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
>> +import org.apache.ctakes.typesystem.type.textspan.Sentence;
>> +import org.apache.uima.analysis_engine.AnalysisEngineDescription;
>> +import org.apache.uima.jcas.JCas;
>> +import org.apache.uima.jcas.tcas.Annotation;
>> +import org.apache.uima.resource.ResourceInitializationException;
>> +import org.cleartk.ml.CleartkAnnotator;
>> +import org.cleartk.ml.DataWriter;
>> +import org.cleartk.ml.jar.DefaultDataWriterFactory;
>> +import org.cleartk.ml.jar.DirectoryDataWriterFactory;
>> +import org.cleartk.ml.jar.GenericJarClassifierFactory;
>> +import org.apache.uima.fit.factory.AnalysisEngineFactory;
>> +import org.apache.uima.fit.util.JCasUtil;
>> +
>> +import com.google.common.collect.Lists;
>> +
>> +public class EventEventI2B2RelationAnnotator extends RelationExtractorAnnotator {
>> +
>> +    public static AnalysisEngineDescription createDataWriterDescription(
>> +            Class<? extends DataWriter<String>> dataWriterClass,
>> +                    File outputDirectory,
>> +                    double probabilityOfKeepingANegativeExample) throws ResourceInitializationException {
>> +        return AnalysisEngineFactory.createEngineDescription(
>> +                EventEventI2B2RelationAnnotator.class,
>> +                CleartkAnnotator.PARAM_IS_TRAINING,
>> +                true,
>> +                DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
>> +                dataWriterClass,
>> +                DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
>> +                outputDirectory,
>> +                RelationExtractorAnnotator.PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
>> +                // not sure why this has to be cast; something funny going on in uimaFIT maybe?
>> +                (float) probabilityOfKeepingANegativeExample);
>> +    }
>> +
>> +    public static AnalysisEngineDescription createAnnotatorDescription(String modelPath)
>> +            throws ResourceInitializationException {
>> +        return AnalysisEngineFactory.createEngineDescription(
>> +                EventEventI2B2RelationAnnotator.class,
>> +                CleartkAnnotator.PARAM_IS_TRAINING,
>> +                false,
>> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
>> +                modelPath);
>> +    }
>> +
>> +    /**
>> +     * @deprecated use String path instead of File.
>> +     * ClearTK will automatically Resolve the String to an InputStream.
>> +     * This will allow resources to be read within from a jar as well as File.  
>> +     */      
>> +    @SuppressWarnings("dep-ann")
>> +    public static AnalysisEngineDescription createAnnotatorDescription(File modelDirectory)
>> +            throws ResourceInitializationException {
>> +        return AnalysisEngineFactory.createEngineDescription(
>> +                EventEventI2B2RelationAnnotator.class,
>> +                CleartkAnnotator.PARAM_IS_TRAINING,
>> +                false,
>> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
>> +                new File(modelDirectory, "model.jar"));
>> +    }
>> +
>> +    @Override
>> +    protected List<RelationFeaturesExtractor> getFeatureExtractors() {
>> +        return Lists.newArrayList(
>> +                //                new TokenFeaturesExtractor()
>> +                new UnexpandedTokenFeaturesExtractor() //use unexpanded version for i2b2 data
>> +                , new PartOfSpeechFeaturesExtractor()
>> +                //                , new TemporalPETExtractor()
>> +                , new EventArgumentPropertyExtractor()
>> +                , new NumberOfEventTimeBetweenCandidatesExtractor()
>> +                , new SectionHeaderRelationExtractor()
>> +                , new NearbyVerbTenseRelationExtractor()
>> +                , new CheckSpecialWordRelationExtractor()
>> +                , new UmlsFeatureExtractor()
>> +                , new DependencyPathFeaturesExtractor()
>> +                , new CoordinateFeaturesExtractor()
>> +                , new OverlappedHeadFeaturesExtractor()
>> +                , new SRLRelationFeaturesExtractor()
>> +                , new NumberOfEventsInTheSameSentenceExtractor()
>> +                , new EventPositionRelationFeaturesExtractor() //not helpful
>> +                , new TimeXRelationFeaturesExtractor() //not helpful
>> +                , new ConjunctionRelationFeaturesExtractor()
>> +                , new DeterminerRelationFeaturesExtractor()
>> +                , new EventTimeRelationFeatureExtractor()
>> +                , new TokenPropertyFeaturesExtractor()
>> +                , new DependingVerbsFeatureExtractor()
>> +                , new SpecialAnnotationRelationExtractor() //not helpful
>> +                , new TemporalPETFlatExtractor()
>> +                //                , new EventInBetweenPropertyExtractor()
>> +                //                , new EventOutsidePropertyExtractor()
>> +                );
>> +    }
>> +
>> +    @Override
>> +    protected Class<? extends Annotation> getCoveringClass() {
>> +        return Sentence.class;
>> +    }
>> +
>> +    @Override
>> +    protected List<IdentifiedAnnotationPair> getCandidateRelationArgumentPairs(
>> +            JCas jCas, Annotation sentence) {
>> +
>> +        Map<EventMention, Collection<EventMention>> coveringMap =
>> +                JCasUtil.indexCovering(jCas, EventMention.class, EventMention.class);
>> +
>> +        List<IdentifiedAnnotationPair> pairs = Lists.newArrayList();
>> +        List<EventMention> events = new ArrayList<>(JCasUtil.selectCovered(jCas, EventMention.class, sentence));
>> +        //filter events:
>> +        List<EventMention> realEvents = Lists.newArrayList();
>> +        for( EventMention event : events){
>> +            if(event.getClass().equals(EventMention.class)){
>> +                realEvents.add(event);
>> +            }
>> +        }
>> +        events = realEvents;
>> +
>> +        int eventNum = events.size();
>> +
>> +        for (int i = 0; i < eventNum-1; i++){
>> +            for(int j = i+1; j < eventNum; j++){
>> +                EventMention eventA = events.get(j);
>> +                EventMention eventB = events.get(i);
>> +
>> +                if(this.isTraining()){
>> +                    //pairing covering system events:
>> +                    for (EventMention event1 : coveringMap.get(eventA)){
>> +                        for(EventMention event2 : coveringMap.get(eventB)){
>> +                            pairs.add(new IdentifiedAnnotationPair(event1, event2));                            
>> +                        }
>> +                        pairs.add(new IdentifiedAnnotationPair(event1, eventB));
>> +                    }
>> +                    for(EventMention event2 : coveringMap.get(eventB)){
>> +                        pairs.add(new IdentifiedAnnotationPair(eventA, event2));                            
>> +                    }
>> +                    //pairing covered system events:
>> +                    for(EventMention event1 : JCasUtil.selectCovered(jCas, EventMention.class, eventA)){
>> +                        for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
>> +                            pairs.add(new IdentifiedAnnotationPair(event1, event2));
>> +                        }
>> +                        pairs.add(new IdentifiedAnnotationPair(event1, eventB));
>> +                    }
>> +                    for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
>> +                        pairs.add(new IdentifiedAnnotationPair(eventA, event2));
>> +                    }
>> +                }
>> +                pairs.add(new IdentifiedAnnotationPair(eventA, eventB));
>> +
>> +            }
>> +        }
>> +
>> +
>> +        //        if(eventNum >= 2){
>> +        //            for ( int i = 0; i< eventNum -1 ; i ++){
>> +        //                EventMention evI = events.get(i);
>> +        //                for(int j = i+1; j< eventNum; j++){
>> +        //                    EventMention evJ = events.get(j);
>> +        //                    if(j-i==1 || j-i==eventNum-1){//if two events are consecutive, or major events
>> +        //                        pairs.add(new IdentifiedAnnotationPair(evJ, evI));
>> +        //                    }else if(ifDependent(jCas, evI, evJ)){//if the event pairs are dependent// eventNum < 7 && 
>> +        //                        pairs.add(new IdentifiedAnnotationPair(evJ, evI));
>> +        //                    }else{// if the 
>> +        //                        continue;
>> +        //                    }
>> +        //                }
>> +        //            }
>> +        //        }
>> +
>> +        return pairs;
>> +    }
>> +
>> +    @Override
>> +    protected void createRelation(JCas jCas, IdentifiedAnnotation arg1,
>> +            IdentifiedAnnotation arg2, String predictedCategory) {
>> +        RelationArgument relArg1 = new RelationArgument(jCas);
>> +        relArg1.setArgument(arg1);
>> +        relArg1.setRole("Arg1");
>> +        relArg1.addToIndexes();
>> +        RelationArgument relArg2 = new RelationArgument(jCas);
>> +        relArg2.setArgument(arg2);
>> +        relArg2.setRole("Arg2");
>> +        relArg2.addToIndexes();
>> +        TemporalTextRelation relation = new TemporalTextRelation(jCas);
>> +        relation.setArg1(relArg1);
>> +        relation.setArg2(relArg2);
>> +        relation.setCategory(predictedCategory);
>> +        relation.addToIndexes();
>> +    }
>> +
>> +    @Override
>> +    protected String getRelationCategory(
>> +            Map<List<Annotation>, BinaryTextRelation> relationLookup,
>> +            IdentifiedAnnotation arg1,
>> +            IdentifiedAnnotation arg2) {
>> +        BinaryTextRelation relation = relationLookup.get(Arrays.asList(arg1, arg2));
>> +        String category = null;
>> +        if (relation != null && relation instanceof TemporalTextRelation) {
>> +            category = relation.getCategory();
>> +        } else {
>> +            relation = relationLookup.get(Arrays.asList(arg2, arg1));
>> +            if (relation != null && relation instanceof TemporalTextRelation) {
>> +                if(relation.getCategory().equals("OVERLAP")){
>> +                    category = relation.getCategory();
>> +                    //                }else if (relation.getCategory().equals("BEFORE")){
>> +                    //                    category = "AFTER";
>> +                    //                }else if (relation.getCategory().equals("AFTER")){
>> +                    //                    category = "BEFORE";
>> +                    //                }
>> +                }else{
>> +                    category = relation.getCategory() + "-1";
>> +                }
>> +            }
>> +        }
>> +        if (category == null && coin.nextDouble() <= this.probabilityOfKeepingANegativeExample) {
>> +            category = NO_RELATION_CATEGORY;
>> +        }
>> +        return category;
>> +    }
>> +}
>> 
>> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java
>> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java?rev=1637884&r1=1637883&r2=1637884&view=diff
>> ==============================================================================
>> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java (original)
>> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventEventRelationAnnotator.java Mon Nov 10 15:19:55 2014
>> @@ -198,15 +198,25 @@ public class EventEventRelationAnnotator
>>               }
>>               if(eventAValid && eventBValid){
>>                   if(this.isTraining()){
>> +                        //pairing covering system events:
>>                       for (EventMention event1 : coveringMap.get(eventA)){
>>                           for(EventMention event2 : coveringMap.get(eventB)){
>>                               pairs.add(new IdentifiedAnnotationPair(event1, event2));
>>                           }
>> +                            pairs.add(new IdentifiedAnnotationPair(event1, eventB));
>>                       }
>> +                        for(EventMention event2 : coveringMap.get(eventB)){
>> +                            pairs.add(new IdentifiedAnnotationPair(eventA, event2));
>> +                        }
>> +                        //pairing covered system events:
>>                       for(EventMention event1 : JCasUtil.selectCovered(jCas, EventMention.class, eventA)){
>>                           for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
>>                               pairs.add(new IdentifiedAnnotationPair(event1, event2));
>>                           }
>> +                            pairs.add(new IdentifiedAnnotationPair(event1, eventB));
>> +                        }
>> +                        for(EventMention event2 : JCasUtil.selectCovered(jCas, EventMention.class, eventB)){
>> +                            pairs.add(new IdentifiedAnnotationPair(eventA, event2));
>>                       }
>>                   }
>>                   pairs.add(new IdentifiedAnnotationPair(eventA, eventB));
>> 
>> Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java
>> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java?rev=1637884&view=auto
>> ==============================================================================
>> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java (added)
>> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeI2B2RelationAnnotator.java Mon Nov 10 15:19:55 2014
>> @@ -0,0 +1,253 @@
>> +/**
>> + * Licensed to the Apache Software Foundation (ASF) under one
>> + * or more contributor license agreements.  See the NOTICE file
>> + * distributed with this work for additional information
>> + * regarding copyright ownership.  The ASF licenses this file
>> + * to you under the Apache License, Version 2.0 (the
>> + * "License"); you may not use this file except in compliance
>> + * with the License.  You may obtain a copy of the License at
>> + *
>> + *   http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing,
>> + * software distributed under the License is distributed on an
>> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
>> + * KIND, either express or implied.  See the License for the
>> + * specific language governing permissions and limitations
>> + * under the License.
>> + */
>> +package org.apache.ctakes.temporal.ae;
>> +
>> +import java.io.File;
>> +import java.util.Arrays;
>> +import java.util.Collection;
>> +import java.util.List;
>> +import java.util.Map;
>> +
>> +import org.apache.ctakes.relationextractor.ae.RelationExtractorAnnotator;
>> +import org.apache.ctakes.relationextractor.ae.features.PartOfSpeechFeaturesExtractor;
>> +import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.CheckSpecialWordRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.ConjunctionRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.DependencyFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.DependencyPathFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.EventArgumentPropertyExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.EventIndexOfSameSentenceRelationFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.EventPositionRelationFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.EventTimeRelationFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.NearbyVerbTenseRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.NearestFlagFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.NumberOfEventTimeBetweenCandidatesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.OverlappedHeadFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.SRLRelationFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.SectionHeaderRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TemporalAttributeFeatureExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TemporalPETFlatExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.TimeWordTypeRelationExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TimeXPropertyRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.TimeXRelationFeaturesExtractor;
>> +import org.apache.ctakes.temporal.ae.feature.UnexpandedTokenFeaturesExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.TemporalAttributeFeatureExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.treekernel.EventTimeFlatTreeFeatureExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.treekernel.EventVerbRelationTreeExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.treekernel.TemporalPETExtractor;
>> +//import org.apache.ctakes.temporal.ae.feature.treekernel.TemporalPathExtractor;
>> +import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
>> +import org.apache.ctakes.typesystem.type.relation.RelationArgument;
>> +import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
>> +import org.apache.ctakes.typesystem.type.textsem.EventMention;
>> +import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
>> +import org.apache.ctakes.typesystem.type.textsem.TimeMention;
>> +import org.apache.ctakes.typesystem.type.textspan.Sentence;
>> +import org.apache.uima.analysis_engine.AnalysisEngineDescription;
>> +import org.apache.uima.jcas.JCas;
>> +import org.apache.uima.jcas.tcas.Annotation;
>> +import org.apache.uima.resource.ResourceInitializationException;
>> +import org.cleartk.ml.CleartkAnnotator;
>> +import org.cleartk.ml.DataWriter;
>> +import org.cleartk.ml.jar.DefaultDataWriterFactory;
>> +import org.cleartk.ml.jar.DirectoryDataWriterFactory;
>> +import org.cleartk.ml.jar.GenericJarClassifierFactory;
>> +import org.apache.uima.fit.factory.AnalysisEngineFactory;
>> +import org.apache.uima.fit.util.JCasUtil;
>> +
>> +import com.google.common.collect.Lists;
>> +
>> +public class EventTimeI2B2RelationAnnotator extends RelationExtractorAnnotator {
>> +
>> +    public static AnalysisEngineDescription createDataWriterDescription(
>> +            Class<? extends DataWriter<String>> dataWriterClass,
>> +                    File outputDirectory,
>> +                    double probabilityOfKeepingANegativeExample) throws ResourceInitializationException {
>> +        return AnalysisEngineFactory.createEngineDescription(
>> +                EventTimeI2B2RelationAnnotator.class,
>> +                CleartkAnnotator.PARAM_IS_TRAINING,
>> +                true,
>> +                DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
>> +                dataWriterClass,
>> +                DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
>> +                outputDirectory,
>> +                RelationExtractorAnnotator.PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
>> +                // not sure why this has to be cast; something funny going on in uimaFIT maybe?
>> +                (float) probabilityOfKeepingANegativeExample);
>> +    }
>> +
>> +    public static AnalysisEngineDescription createEngineDescription(String modelPath)
>> +            throws ResourceInitializationException {
>> +        return AnalysisEngineFactory.createEngineDescription(
>> +                EventTimeI2B2RelationAnnotator.class,
>> +                CleartkAnnotator.PARAM_IS_TRAINING,
>> +                false,
>> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
>> +                modelPath);
>> +    }
>> +    /**
>> +     * @deprecated use String path instead of File.
>> +     * ClearTK will automatically Resolve the String to an InputStream.
>> +     * This will allow resources to be read within from a jar as well as File.  
>> +     */     
>> +    @Deprecated
>> +    public static AnalysisEngineDescription createEngineDescription(File modelDirectory)
>> +            throws ResourceInitializationException {
>> +        return AnalysisEngineFactory.createEngineDescription(
>> +                EventTimeI2B2RelationAnnotator.class,
>> +                CleartkAnnotator.PARAM_IS_TRAINING,
>> +                false,
>> +                GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH,
>> +                new File(modelDirectory, "model.jar"));
>> +    }
>> +
>> +    @Override
>> +    protected List<RelationFeaturesExtractor> getFeatureExtractors() {
>> +        return Lists.newArrayList(
>> +//                new TokenFeaturesExtractor()
>> +                new UnexpandedTokenFeaturesExtractor() //use unexpanded version for i2b2 data
>> +                , new PartOfSpeechFeaturesExtractor()
>> +                , new TemporalAttributeFeatureExtractor()
>> +                //                , new EventTimeFlatTreeFeatureExtractor()
>> +                //                , new TemporalPETExtractor()
>> +                //, new TemporalPathExtractor()
>> +                //                , new EventVerbRelationTreeExtractor()
>> +                , new NumberOfEventTimeBetweenCandidatesExtractor()
>> +                //                , new SectionHeaderRelationExtractor()
>> +                , new NearbyVerbTenseRelationExtractor()
>> +                , new CheckSpecialWordRelationExtractor()
>> +                , new NearestFlagFeatureExtractor()
>> +                , new DependencyPathFeaturesExtractor()
>> +                , new DependencyFeatureExtractor()
>> +                //                , new SRLRelationFeaturesExtractor()// tried, but not helpful
>> +                , new EventArgumentPropertyExtractor()
>> +                , new OverlappedHeadFeaturesExtractor()
>> +                //                , new EventTimeRelationFeatureExtractor()
>> +                , new ConjunctionRelationFeaturesExtractor()
>> +                //                , new EventPositionRelationFeaturesExtractor() //tried, but not helpful
>> +                , new TimeXRelationFeaturesExtractor()
>> +                , new TemporalPETFlatExtractor()
>> +                , new TimeXPropertyRelationFeaturesExtractor()
>> +                //                , new TimeWordTypeRelationExtractor() //tried, but not helpful
>> +                //                , new EventIndexOfSameSentenceRelationFeaturesExtractor() //tried, but not helpful
>> +                );
>> +    }
>> +
>> +    @Override
>> +    protected Class<? extends Annotation> getCoveringClass() {
>> +        return Sentence.class;
>> +    }
>> +
>> +    @Override
>> +    public List<IdentifiedAnnotationPair> getCandidateRelationArgumentPairs(
>> +            JCas jCas,
>> +            Annotation sentence) {
>> +        Map<EventMention, Collection<EventMention>> coveringMap =
>> +                JCasUtil.indexCovering(jCas, EventMention.class, EventMention.class);
>> +
>> +        List<IdentifiedAnnotationPair> pairs = Lists.newArrayList();
>> +        for (EventMention event : JCasUtil.selectCovered(jCas, EventMention.class, sentence)) {
>> +            boolean eventValid = false;
>> +            if (event.getClass().equals(EventMention.class)) {//event is a gold event
>> +                eventValid = true;
>> +            }
>> +
>> +            if(eventValid){
>> +                // ignore subclasses like Procedure and Disease/Disorder
>> +                if(this.isTraining()){//if training mode, train on both gold event and span-overlapping system events
>> +                    for (TimeMention time : JCasUtil.selectCovered(jCas, TimeMention.class, sentence)) {
>> +                        Collection<EventMention> eventList = coveringMap.get(event);
>> +                        for(EventMention covEvent : eventList){
>> +                            pairs.add(new IdentifiedAnnotationPair(covEvent, time));
>> +                        }
>> +                        pairs.add(new IdentifiedAnnotationPair(event, time));
>> +                        for(EventMention covedEvent : JCasUtil.selectCovered(jCas, EventMention.class, event)){//select covered events
>> +                            pairs.add(new IdentifiedAnnotationPair(covedEvent, time));
>> +                        }
>> +                    }
>> +                }else{//if testing mode, only test on system generated events
>> +                    for (TimeMention time : JCasUtil.selectCovered(jCas, TimeMention.class, sentence)) {
>> +                        pairs.add(new IdentifiedAnnotationPair(event, time));
>> +                    }
>> +                }
>> +            }
>> +        }
>> +
>> +        return pairs;
>> +    }
>> +
>> +    @Override
>> +    protected void createRelation(JCas jCas, IdentifiedAnnotation arg1,
>> +            IdentifiedAnnotation arg2, String predictedCategory) {
>> +        RelationArgument relArg1 = new RelationArgument(jCas);
>> +        relArg1.setArgument(arg1);
>> +        relArg1.setRole("Arg1");
>> +        relArg1.addToIndexes();
>> +        RelationArgument relArg2 = new RelationArgument(jCas);
>> +        relArg2.setArgument(arg2);
>> +        relArg2.setRole("Arg2");
>> +        relArg2.addToIndexes();
>> +        TemporalTextRelation relation = new TemporalTextRelation(jCas);
>> +        relation.setArg1(relArg1);
>> +        relation.setArg2(relArg2);
>> +        relation.setCategory(predictedCategory);
>> +        relation.addToIndexes();
>> +    }
>> +
>> +
>> +    @Override
>> +    protected String getRelationCategory(
>> +            Map<List<Annotation>, BinaryTextRelation> relationLookup,
>> +            IdentifiedAnnotation arg1,
>> +            IdentifiedAnnotation arg2) {
>> +        BinaryTextRelation relation = relationLookup.get(Arrays.asList(arg1, arg2));
>> +        String category = null;
>> +        if (relation != null) {
>> +            category = relation.getCategory();
>> +        } else {
>> +            relation = relationLookup.get(Arrays.asList(arg2, arg1));
>> +            if (relation != null) {
>> +                if(relation.getCategory().equals("OVERLAP")){
>> +                    category = relation.getCategory();
>> +                    //                }else if (relation.getCategory().equals("BEFORE")){
>> +                    //                    category = "AFTER";
>> +                    //                }else if (relation.getCategory().equals("AFTER")){
>> +                    //                    category = "BEFORE";
>> +                    //                }
>> +                }else{
>> +                    category = relation.getCategory() + "-1";
>> +                }
>> +            }
>> +        }
>> +
>> +        if(category!=null){
>> +            if(!((EventMention)arg1).getClass().equals(EventMention.class)){
>> +                System.out.println("find system-event relations: "+ arg1.getCoveredText() + " -"+category+"- " + arg2.getCoveredText());
>> +            }else{
>> +                System.out.println("find gold-event relations: "+ arg1.getCoveredText() + " -"+category+"- " + arg2.getCoveredText());
>> +            }
>> +        }
>> +
>> +        if (category == null && coin.nextDouble() <= this.probabilityOfKeepingANegativeExample) {
>> +            category = NO_RELATION_CATEGORY;
>> +        }
>> +
>> +        return category;
>> +    }
>> +}
>> 
>> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java
>> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java?rev=1637884&r1=1637883&r2=1637884&view=diff
>> ==============================================================================
>> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java (original)
>> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java Mon Nov 10 15:19:55 2014
>> @@ -329,7 +329,7 @@ EvaluationOfTemporalRelations_ImplBase{
>>       }
>> 
>>       //    HideOutput hider = new HideOutput();
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"),"-c", optArray[1]);//"-w1","0.09","-w2","4","-w3","9","-w5","2","-w6","16","-w7","10","-w8","6", "-w9","45","-w10","30","-c", optArray[1]);//"-c", "0.05");//optArray);
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"),"-w1","0.08","-w3","3","-w4","17","-w5","20","-w6","16","-w7","10","-w8","6", "-w9","45","-w10","30","-c", optArray[1]);//"-c", "0.05");//optArray);
>>       //        JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"), "-h","0","-c", "1000");
>>       //    hider.restoreOutput();
>>       //    hider.close();
>> @@ -529,6 +529,20 @@ EvaluationOfTemporalRelations_ImplBase{
>>                       for(EventMention covEventB : coveringMap.get(event2)){
>>                           createRelation(relationView, covEventA, covEventB, relation.getCategory());
>>                       }
>> +                        createRelation(relationView, covEventA, event2, relation.getCategory());
>> +                    }
>> +                    for(EventMention covEventB : coveringMap.get(event2)){
>> +                        createRelation(relationView, event1, covEventB, relation.getCategory());
>> +                    }
>> +                    //get covered system events:
>> +                    for(EventMention covedEventA : JCasUtil.selectCovered(jCas, EventMention.class, event1)){//select covered events
>> +                        for(EventMention covedEventB : JCasUtil.selectCovered(jCas, EventMention.class, event2)){
>> +                            createRelation(relationView, covedEventA, covedEventB, relation.getCategory());
>> +                        }
>> +                        createRelation(relationView, covedEventA, event2, relation.getCategory());
>> +                    }
>> +                    for(EventMention covedEventB : JCasUtil.selectCovered(jCas, EventMention.class, event2)){
>> +                        createRelation(relationView, event1, covedEventB, relation.getCategory());
>>                   }
>>               }
>>           }
>> 
>> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
>> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java?rev=1637884&r1=1637883&r2=1637884&view=diff
>> ==============================================================================
>> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java (original)
>> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java Mon Nov 10 15:19:55 2014
>> @@ -34,6 +34,7 @@ import java.util.List;
>> import java.util.Map;
>> import java.util.Set;
>> 
>> +import org.apache.ctakes.relationextractor.ae.RelationExtractorAnnotator.IdentifiedAnnotationPair;
>> import org.apache.ctakes.relationextractor.eval.RelationExtractorEvaluation.HashableArguments;
>> import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
>> import org.apache.ctakes.temporal.ae.EventTimeSelfRelationAnnotator;
>> @@ -693,6 +694,9 @@ EvaluationOfTemporalRelations_ImplBase{
>>                           createRelation(relationView, covEvent, arg2, relation.getCategory());
>>                       }
>>                   }
>> +                    for(EventMention covedEvent : JCasUtil.selectCovered(jCas, EventMention.class, event)){//select covered events
>> +                        createRelation(relationView, covedEvent, arg2, relation.getCategory());
>> +                    }
>>               }else if(arg2 instanceof EventMention && arg1 instanceof TimeMention){
>>                   event = (EventMention) arg2;
>>                   Collection<EventMention> eventList = coveringMap.get(event);
>> @@ -701,6 +705,9 @@ EvaluationOfTemporalRelations_ImplBase{
>>                           createRelation(relationView, arg1, covEvent, relation.getCategory());
>>                       }
>>                   }
>> +                    for(EventMention covedEvent : JCasUtil.selectCovered(jCas, EventMention.class, event)){//select covered events
>> +                        createRelation(relationView, arg1, covedEvent, relation.getCategory());
>> +                    }
>>               }
>>           }
>> 
>> 
>> Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java
>> URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java?rev=1637884&r1=1637883&r2=1637884&view=diff
>> ==============================================================================
>> --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java (original)
>> +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfI2B2TemporalRelations.java Mon Nov 10 15:19:55 2014
>> @@ -38,14 +38,15 @@ import org.apache.ctakes.core.ae.CDASegm
>> import org.apache.ctakes.relationextractor.eval.RelationExtractorEvaluation.HashableArguments;
>> import org.apache.ctakes.temporal.ae.ConsecutiveSentencesEventEventRelationAnnotator;
>> import org.apache.ctakes.temporal.ae.ConsecutiveSentencesEventTimeRelationAnnotator;
>> -import org.apache.ctakes.temporal.ae.EventTimeSelfRelationAnnotator;
>> import org.apache.ctakes.temporal.ae.TemporalRelationRuleAnnotator;
>> import org.apache.ctakes.temporal.ae.DocTimeRelAnnotator;
>> import org.apache.ctakes.temporal.ae.EventAdmissionTimeAnnotator;
>> import org.apache.ctakes.temporal.ae.EventDischargeTimeAnnotator;
>> -import org.apache.ctakes.temporal.ae.EventEventRelationAnnotator;
>> -import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
>> +import org.apache.ctakes.temporal.ae.EventEventI2B2RelationAnnotator;
>> +import org.apache.ctakes.temporal.ae.EventTimeI2B2RelationAnnotator;
>> import org.apache.ctakes.temporal.ae.baselines.RecallBaselineEventTimeRelationAnnotator;
>> +import org.apache.ctakes.temporal.eval.EvaluationOfEventEventThymeRelations.AddEEPotentialRelations;
>> +import org.apache.ctakes.temporal.eval.EvaluationOfEventTimeRelations.AddPotentialRelations;
>> import org.apache.ctakes.temporal.eval.EvaluationOfEventTimeRelations.ParameterSettings;
>> import org.apache.ctakes.temporal.eval.EvaluationOfTemporalRelations_ImplBase.RemoveNonContainsRelations.RemoveGoldAttributes;
>> import org.apache.ctakes.temporal.utils.AnnotationIdCollection;
>> @@ -66,7 +67,6 @@ import org.apache.uima.fit.component.JCa
>> import org.apache.uima.fit.descriptor.ConfigurationParameter;
>> import org.apache.uima.fit.factory.AggregateBuilder;
>> import org.apache.uima.fit.factory.AnalysisEngineFactory;
>> -import org.apache.uima.fit.pipeline.JCasIterator;
>> import org.apache.uima.fit.pipeline.SimplePipeline;
>> import org.apache.uima.fit.util.JCasUtil;
>> import org.apache.uima.jcas.JCas;
>> @@ -75,7 +75,7 @@ import org.apache.uima.resource.Resource
>> import org.apache.uima.util.FileUtils;
>> import org.cleartk.eval.AnnotationStatistics;
>> import org.cleartk.ml.jar.JarClassifierBuilder;
>> -//import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter;
>> +import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter;
>> import org.cleartk.ml.liblinear.LibLinearStringOutcomeDataWriter;
>> //import org.cleartk.classifier.tksvmlight.TKSVMlightStringOutcomeDataWriter;
>> import org.cleartk.ml.tksvmlight.model.CompositeKernel.ComboOperator;
>> @@ -292,13 +292,18 @@ EvaluationOfTemporalRelations_ImplBase{
>>       //                CAS.NAME_DEFAULT_SOFA,
>>       //                GOLD_VIEW_NAME);
>>       //        aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveNonContainsRelations.class));
>> -        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
>> +//        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
>> 
>>       //        aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveEventEventRelations.class));
>>       //test rules:
>>       //        aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(TemporalRelationRuleAnnotator.class));
>> +        
>> +        //add unlabeled nearby system events as potential event-time links: 
>> +        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddPotentialRelations.class));
>> +        //add unlabeled nearby system events as potential event-event links: 
>> +        aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddEEPotentialRelations.class));
>> 
>> -        aggregateBuilder.add(EventTimeSelfRelationAnnotator.createDataWriterDescription(
>> +        aggregateBuilder.add(EventTimeI2B2RelationAnnotator.createDataWriterDescription(
>>               //                LibSvmStringOutcomeDataWriter.class,
>>               LibLinearStringOutcomeDataWriter.class,
>>               // TKSVMlightStringOutcomeDataWriter.class,
>> @@ -306,27 +311,27 @@ EvaluationOfTemporalRelations_ImplBase{
>>               //        SVMlightStringOutcomeDataWriter.class,        
>>               new File(directory,EVENT_TIME),
>>               params.probabilityOfKeepingANegativeExample));
>> -        aggregateBuilder.add(EventEventRelationAnnotator.createDataWriterDescription(
>> +        aggregateBuilder.add(EventEventI2B2RelationAnnotator.createDataWriterDescription(
>>               LibLinearStringOutcomeDataWriter.class,//TKSVMlightStringOutcomeDataWriter.class,//
>>               //                LIBLINEARStringOutcomeDataWriter.class,
>>               new File(directory,EVENT_EVENT), 
>>               params.probabilityOfKeepingANegativeExample));
>>       aggregateBuilder.add(EventDischargeTimeAnnotator.createDataWriterDescription(
>> -                //                LibSvmStringOutcomeDataWriter.class,
>> -                LibLinearStringOutcomeDataWriter.class,
>> +                LibSvmStringOutcomeDataWriter.class,
>> +//                LibLinearStringOutcomeDataWriter.class,
>>               new File(directory,EVENT_DISCHARGE)));
>>       aggregateBuilder.add(EventAdmissionTimeAnnotator.createDataWriterDescription(
>> -                //                LibSvmStringOutcomeDataWriter.class,
>> -                LibLinearStringOutcomeDataWriter.class,
>> +                LibSvmStringOutcomeDataWriter.class,
>> +//                LibLinearStringOutcomeDataWriter.class,
>>               new File(directory,EVENT_ADMISSION)));
>>       aggregateBuilder.add(ConsecutiveSentencesEventEventRelationAnnotator.createDataWriterDescription(
>> -                //                LibSvmStringOutcomeDataWriter.class,
>> -                LibLinearStringOutcomeDataWriter.class,
>> +                LibSvmStringOutcomeDataWriter.class,
>> +//                LibLinearStringOutcomeDataWriter.class,
>>               new File(directory,TEMP_CROSSSENT), 
>>               params.probabilityOfKeepingANegativeExample));
>>       aggregateBuilder.add(ConsecutiveSentencesEventTimeRelationAnnotator.createDataWriterDescription(
>> -                //                LibSvmStringOutcomeDataWriter.class,
>> -                LibLinearStringOutcomeDataWriter.class,
>> +                LibSvmStringOutcomeDataWriter.class,
>> +//                LibLinearStringOutcomeDataWriter.class,
>>               new File(directory,TEMPET_CROSSSENT), 
>>               params.probabilityOfKeepingANegativeExample));
>>       SimplePipeline.runPipeline(collectionReader, aggregateBuilder.createAggregate());
>> @@ -355,14 +360,14 @@ EvaluationOfTemporalRelations_ImplBase{
>>       }
>> 
>>       //    HideOutput hider = new HideOutput();
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_TIME), "-c", "0.001");//"-h","0","-c", "1000");//optArray);//"-c", "0.05");//
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_EVENT), "-c", "0.001");
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_DISCHARGE), "-c", "0.001");
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_ADMISSION), "-c", "0.001");
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_TIME), "-c", "0.0002", "-w2","0.2","-w3","7","-w4","7","-w5","13");//"-h","0","-c", "1000");//optArray);//"-c", "0.05");//
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_EVENT), "-c", "0.0002","-w1","0.07","-w2","14","-w3","5","-w4","10","-w6","20");
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_DISCHARGE), "-h","0","-c", "1000");
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,EVENT_ADMISSION), "-h","0","-c", "1000");
>>       //        JarClassifierBuilder.trainAndPackage(new File(directory,TIME_ADMISSION), "-h","0","-c", "1000");
>>       //        JarClassifierBuilder.trainAndPackage(new File(directory,TIME_DISCHARGE), "-h","0","-c", "1000");
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,TEMP_CROSSSENT), "-c", "0.001");
>> -        JarClassifierBuilder.trainAndPackage(new File(directory,TEMPET_CROSSSENT), "-c", "0.001");
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,TEMP_CROSSSENT), "-h","0","-c", "1000");
>> +        JarClassifierBuilder.trainAndPackage(new File(directory,TEMPET_CROSSSENT), "-h","0","-c", "1000");
>>       //    hider.restoreOutput();
>>       //    hider.close();
>>   }
>> @@ -419,8 +424,8 @@ EvaluationOfTemporalRelations_ImplBase{
>>       //        aggregateBuilder.add(TimexDischargeTimeAnnotator.createAnnotatorDescription(new File(directory,TIME_DISCHARGE)));
>> 
>>       aggregateBuilder.add(this.baseline ? RecallBaselineEventTimeRelationAnnotator.createAnnotatorDescription(directory) :
>> -            EventTimeSelfRelationAnnotator.createEngineDescription(new File(directory,EVENT_TIME)));
>> -        aggregateBuilder.add(EventEventRelationAnnotator.createAnnotatorDescription(new File(directory,EVENT_EVENT)));
>> +            EventTimeI2B2RelationAnnotator.createEngineDescription(new File(directory,EVENT_TIME)));
>> +        aggregateBuilder.add(EventEventI2B2RelationAnnotator.createAnnotatorDescription(new File(directory,EVENT_EVENT)));
>>       aggregateBuilder.add(ConsecutiveSentencesEventEventRelationAnnotator.createAnnotatorDescription(new File(directory,TEMP_CROSSSENT)));
>>       aggregateBuilder.add(ConsecutiveSentencesEventTimeRelationAnnotator.createAnnotatorDescription(new File(directory,TEMPET_CROSSSENT)));
>>       aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(TemporalRelationRuleAnnotator.class));
>> 
>>