You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by jo...@apache.org on 2009/05/30 13:13:01 UTC
svn commit: r780236 [4/6] - in /incubator/uima/sandbox/trunk/Lucas: ./ src/
src/main/ src/main/java/ src/main/java/org/ src/main/java/org/apache/
src/main/java/org/apache/uima/ src/main/java/org/apache/uima/lucas/
src/main/java/org/apache/uima/lucas/co...
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStreamTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStreamTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStreamTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStreamTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,1438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.uima.UIMAFramework;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.CASException;
+import org.apache.uima.collection.CollectionReader;
+import org.apache.uima.collection.CollectionReaderDescription;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.cas.FSArray;
+import org.apache.uima.jcas.cas.StringArray;
+import org.apache.uima.lucas.indexer.analysis.AnnotationTokenStream;
+import org.apache.uima.lucas.indexer.types.test.Annotation1;
+import org.apache.uima.lucas.indexer.types.test.FeatureStructure1;
+import org.apache.uima.lucas.indexer.types.test.FeatureStructure2;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.CasCreationUtils;
+import org.apache.uima.util.XMLInputSource;
+
+public class AnnotationTokenStreamTest extends TestCase {
+
+ private final static String READER_DESCRIPTOR =
+ "src/test/resources/AnnotationTokenStreamTestDummyCollectionReader.xml";
+
+ private CollectionReader reader;
+
+ @Override
+ protected void setUp() throws Exception {
+ CollectionReaderDescription readerDescription =
+ (CollectionReaderDescription) UIMAFramework.getXMLParser()
+ .parseCollectionReaderDescription(new XMLInputSource(READER_DESCRIPTOR));
+ reader = UIMAFramework.produceCollectionReader(readerDescription);
+ super.setUp();
+ }
+
+ public JCas createCasWithText(String documentText) throws ResourceInitializationException,
+ CASException {
+ JCas cas = CasCreationUtils.createCas(reader.getProcessingResourceMetaData()).getJCas();
+ cas.setDocumentText(documentText);
+ return cas;
+ }
+
+ public void testCreateDocumentTextTokens() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ annotation1.addToIndexes();
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ annotation3.addToIndexes();
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1");
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateFeatureTokens() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ annotation1.setFeatureString("token1Feature1");
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setFeatureString("token2Feature1");
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setFeatureString("token3Feature1");
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ annotation3.addToIndexes();
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureString", null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1Feature1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2Feature1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3Feature1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateMultipleFeatureTokens() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ annotation1.setFeatureString("token1Feature1");
+ annotation1.setFeatureInteger(1);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setFeatureString("token2Feature1");
+ annotation2.setFeatureInteger(2);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setFeatureString("token3Feature1");
+ annotation3.setFeatureInteger(3);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("featureString");
+ featureNames.add("featureInteger");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", featureNames, "_", null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1Feature1_1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2Feature1_2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3Feature1_3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateArrayFeatureTokensWithoutDelimiter() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ StringArray stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token1FeatureStringArray1");
+ stringArray1.set(1, "token1FeatureStringArray2");
+ stringArray1.set(2, "token1FeatureStringArray3");
+ annotation1.setFeatureStringArray(stringArray1);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ StringArray stringArray2 = new StringArray(cas, 3);
+ stringArray2.set(0, "token2FeatureStringArray1");
+ stringArray2.set(1, "token2FeatureStringArray2");
+ stringArray2.set(2, "token2FeatureStringArray3");
+ annotation2.setFeatureStringArray(stringArray2);
+
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ StringArray stringArray3 = new StringArray(cas, 3);
+ stringArray3.set(0, "token3FeatureStringArray1");
+ stringArray3.set(1, "token3FeatureStringArray2");
+ stringArray3.set(2, "token3FeatureStringArray3");
+ annotation3.setFeatureStringArray(stringArray3);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("featureStringArray");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", featureNames, null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStringArray1", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStringArray2", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStringArray3", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStringArray1", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStringArray2", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStringArray3", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStringArray1", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStringArray2", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStringArray3", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateArrayFeatureTokensWithDelimiter() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ StringArray stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token1FeatureStringArray1");
+ stringArray1.set(1, "token1FeatureStringArray2");
+ stringArray1.set(2, "token1FeatureStringArray3");
+ annotation1.setFeatureStringArray(stringArray1);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ StringArray stringArray2 = new StringArray(cas, 3);
+ stringArray2.set(0, "token2FeatureStringArray1");
+ stringArray2.set(1, "token2FeatureStringArray2");
+ stringArray2.set(2, "token2FeatureStringArray3");
+ annotation2.setFeatureStringArray(stringArray2);
+
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ StringArray stringArray3 = new StringArray(cas, 3);
+ stringArray3.set(0, "token3FeatureStringArray1");
+ stringArray3.set(1, "token3FeatureStringArray2");
+ stringArray3.set(2, "token3FeatureStringArray3");
+ annotation3.setFeatureStringArray(stringArray3);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("featureStringArray");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", featureNames, " ", null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStringArray1 token1FeatureStringArray2 token1FeatureStringArray3",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStringArray1 token2FeatureStringArray2 token2FeatureStringArray3",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStringArray1 token3FeatureStringArray2 token3FeatureStringArray3",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateFeatureStructureTokens() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ FeatureStructure1 featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1Feature1");
+ FeatureStructure2 featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token1Feature3Feature1");
+ featureStructure1.setFeature3(featureStructure2);
+ annotation1.setFeatureStructure1(featureStructure1);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2Feature1");
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token2Feature3Feature1");
+ featureStructure1.setFeature3(featureStructure2);
+ annotation2.setFeatureStructure1(featureStructure1);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3Feature1");
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token3Feature3Feature1");
+ featureStructure1.setFeature3(featureStructure2);
+ annotation3.setFeatureStructure1(featureStructure1);
+ annotation3.addToIndexes();
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("feature1");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureStructure1", featureNames,
+ null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1Feature1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2Feature1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3Feature1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureStructure1.feature3",
+ featureNames, null);
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1Feature3Feature1", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2Feature3Feature1", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3Feature3Feature1", new String(nextToken.termBuffer(), 0, nextToken
+ .termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateArrayFeatureStructuresWithDelimterTokens() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+
+ FSArray featureStructure1Array = new FSArray(cas, 3);
+ FeatureStructure1 featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature10");
+ featureStructure1.setFeature2("token1FeatureStructures1Feature20");
+
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature11");
+ featureStructure1.setFeature2("token1FeatureStructures1Feature21");
+
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature12");
+ featureStructure1.setFeature2("token1FeatureStructures1Feature22");
+
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation1.setFeatureStructures1(featureStructure1Array);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2FeatureStructures1Feature10");
+ featureStructure1.setFeature2("token2FeatureStructures1Feature20");
+
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2FeatureStructures1Feature11");
+ featureStructure1.setFeature2("token2FeatureStructures1Feature21");
+
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2FeatureStructures1Feature12");
+ featureStructure1.setFeature2("token2FeatureStructures1Feature22");
+
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation2.setFeatureStructures1(featureStructure1Array);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature10");
+ featureStructure1.setFeature2("token3FeatureStructures1Feature20");
+
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature11");
+ featureStructure1.setFeature2("token3FeatureStructures1Feature21");
+
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature12");
+ featureStructure1.setFeature2("token3FeatureStructures1Feature22");
+
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation3.setFeatureStructures1(featureStructure1Array);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("feature1");
+ featureNames.add("feature2");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureStructures1", featureNames,
+ ", ", null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature10, token1FeatureStructures1Feature20",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature11, token1FeatureStructures1Feature21",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature12, token1FeatureStructures1Feature22",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature10, token2FeatureStructures1Feature20",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature11, token2FeatureStructures1Feature21",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature12, token2FeatureStructures1Feature22",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature10, token3FeatureStructures1Feature20",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature11, token3FeatureStructures1Feature21",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature12, token3FeatureStructures1Feature22",
+ new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ // public void testCreateArrayFeatureStructuresWithoutDelimterTokens() throws Exception{
+ // JCas cas = createCasWithText("token1 token2 token3");
+ // Annotation1 annotation1 = new Annotation1(cas);
+ // annotation1.setBegin(0);
+ // annotation1.setEnd(6);
+ //
+ // FSArray featureStructure1Array = new FSArray(cas, 3);
+ // FeatureStructure1 featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token1FeatureStructures1Feature10");
+ // featureStructure1.setFeature2("token1FeatureStructures1Feature20");
+ //
+ // featureStructure1Array.set(0, featureStructure1);
+ //
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token1FeatureStructures1Feature11");
+ // featureStructure1.setFeature2("token1FeatureStructures1Feature21");
+ //
+ // featureStructure1Array.set(1, featureStructure1);
+ //
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token1FeatureStructures1Feature12");
+ // featureStructure1.setFeature2("token1FeatureStructures1Feature22");
+ //
+ // featureStructure1Array.set(2, featureStructure1);
+ //
+ // annotation1.setFeatureStructures1(featureStructure1Array);
+ // annotation1.addToIndexes();
+ //
+ // Annotation1 annotation2 = new Annotation1(cas);
+ // annotation2.setBegin(7);
+ // annotation2.setEnd(13);
+ //
+ // featureStructure1Array = new FSArray(cas, 3);
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token2FeatureStructures1Feature10");
+ // featureStructure1.setFeature2("token2FeatureStructures1Feature20");
+ //
+ // featureStructure1Array.set(0, featureStructure1);
+ //
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token2FeatureStructures1Feature11");
+ // featureStructure1.setFeature2("token2FeatureStructures1Feature21");
+ //
+ // featureStructure1Array.set(1, featureStructure1);
+ //
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token2FeatureStructures1Feature12");
+ // featureStructure1.setFeature2("token2FeatureStructures1Feature22");
+ //
+ // featureStructure1Array.set(2, featureStructure1);
+ //
+ // annotation2.setFeatureStructures1(featureStructure1Array);
+ // annotation2.addToIndexes();
+ //
+ // Annotation1 annotation3 = new Annotation1(cas);
+ // annotation3.setBegin(14);
+ // annotation3.setEnd(20);
+ //
+ // featureStructure1Array = new FSArray(cas, 3);
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token3FeatureStructures1Feature10");
+ // featureStructure1.setFeature2("token3FeatureStructures1Feature20");
+ //
+ // featureStructure1Array.set(0, featureStructure1);
+ //
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token3FeatureStructures1Feature11");
+ // featureStructure1.setFeature2("token3FeatureStructures1Feature21");
+ //
+ // featureStructure1Array.set(1, featureStructure1);
+ //
+ // featureStructure1 = new FeatureStructure1(cas);
+ // featureStructure1.setFeature1("token3FeatureStructures1Feature12");
+ // featureStructure1.setFeature2("token3FeatureStructures1Feature22");
+ //
+ // featureStructure1Array.set(2, featureStructure1);
+ //
+ // annotation3.setFeatureStructures1(featureStructure1Array);
+ // annotation3.addToIndexes();
+ //
+ // List<String> featureNames = new ArrayList<String>();
+ // featureNames.add("feature1");
+ // featureNames.add("feature2");
+ //
+ //
+ // AnnotationTokenStream annotationTokenStream = new AnnotationTokenStream(cas,
+ // "de.julielab.jules.types.test.Annotation1", "featureStructures1", featureNames, null);
+ // Token nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token1FeatureStructures1Feature10", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(0, nextToken.startOffset());
+ // assertEquals(6, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token1FeatureStructures1Feature11", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(0, nextToken.startOffset());
+ // assertEquals(6, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token1FeatureStructures1Feature12", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(0, nextToken.startOffset());
+ // assertEquals(6, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token2FeatureStructures1Feature10", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(7, nextToken.startOffset());
+ // assertEquals(13, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token2FeatureStructures1Feature11", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(7, nextToken.startOffset());
+ // assertEquals(13, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token2FeatureStructures1Feature12", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(7, nextToken.startOffset());
+ // assertEquals(13, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token3FeatureStructures1Feature10", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(14, nextToken.startOffset());
+ // assertEquals(20, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token3FeatureStructures1Feature11", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(14, nextToken.startOffset());
+ // assertEquals(20, nextToken.endOffset());
+ //
+ // nextToken = annotationTokenStream.next();
+ // assertNotNull(nextToken);
+ // assertEquals("token3FeatureStructures1Feature12", new String(nextToken.termBuffer(), 0,
+ // nextToken.termLength()));
+ // assertEquals(14, nextToken.startOffset());
+ // assertEquals(20, nextToken.endOffset());
+ // }
+
+ public void testCreateArrayFeatureStructuresWithSinleFeature() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+
+ FSArray featureStructure1Array = new FSArray(cas, 3);
+
+ FeatureStructure1 featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature10");
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature11");
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature12");
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation1.setFeatureStructures1(featureStructure1Array);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+
+ featureStructure1Array = new FSArray(cas, 3);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2FeatureStructures1Feature10");
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2FeatureStructures1Feature11");
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token2FeatureStructures1Feature12");
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation2.setFeatureStructures1(featureStructure1Array);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature10");
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature11");
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature12");
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation3.setFeatureStructures1(featureStructure1Array);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("feature1");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureStructures1", featureNames,
+ null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature10", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature11", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature12", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature10", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature11", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature12", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature10", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature11", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature12", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateArrayFeatureWithFeaturePath() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+
+ FSArray featureStructure1Array = new FSArray(cas, 3);
+ FeatureStructure1 featureStructure1 = new FeatureStructure1(cas);
+
+ FeatureStructure2 featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token1FeatureStructures1Feature3Feature10");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token1FeatureStructures1Feature3Feature11");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token1FeatureStructures1Feature3Feature12");
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation1.setFeatureStructures1(featureStructure1Array);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token2FeatureStructures1Feature3Feature10");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token2FeatureStructures1Feature3Feature11");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token2FeatureStructures1Feature3Feature12");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation2.setFeatureStructures1(featureStructure1Array);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token3FeatureStructures1Feature3Feature10");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token3FeatureStructures1Feature3Feature11");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token3FeatureStructures1Feature12");
+ featureStructure1.setFeature2("token3FeatureStructures1Feature22");
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token3FeatureStructures1Feature3Feature12");
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation3.setFeatureStructures1(featureStructure1Array);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("feature1");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureStructures1.feature3",
+ featureNames, null);
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature10", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature11", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature12", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature10", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature11", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature12", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature10", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature11", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature12", new String(nextToken.termBuffer(), 0,
+ nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+
+ public void testCreateArrayFeaturesWithFeaturePathAndStringArray() throws Exception {
+
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+
+ FSArray featureStructure1Array = new FSArray(cas, 3);
+ FeatureStructure1 featureStructure1 = new FeatureStructure1(cas);
+ featureStructure1.setFeature1("token1FeatureStructures1Feature10");
+ featureStructure1.setFeature2("token1FeatureStructures1Feature20");
+ StringArray stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token1FeatureStructures1Feature3Feature3_0_0");
+ stringArray1.set(1, "token1FeatureStructures1Feature3Feature3_0_1");
+ stringArray1.set(2, "token1FeatureStructures1Feature3Feature3_0_2");
+
+ FeatureStructure2 featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature1("token1FeatureStructures1Feature3Feature10");
+ featureStructure2.setFeature3(stringArray1);
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token1FeatureStructures1Feature3Feature3_1_0");
+ stringArray1.set(1, "token1FeatureStructures1Feature3Feature3_1_1");
+ stringArray1.set(2, "token1FeatureStructures1Feature3Feature3_1_2");
+
+ featureStructure2 = new FeatureStructure2(cas);
+
+ featureStructure2.setFeature3(stringArray1);
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token1FeatureStructures1Feature3Feature3_2_0");
+ stringArray1.set(1, "token1FeatureStructures1Feature3Feature3_2_1");
+ stringArray1.set(2, "token1FeatureStructures1Feature3Feature3_2_2");
+ featureStructure2 = new FeatureStructure2(cas);
+
+ featureStructure2.setFeature3(stringArray1);
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation1.setFeatureStructures1(featureStructure1Array);
+ annotation1.addToIndexes();
+
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token2FeatureStructures1Feature3Feature3_0_0");
+ stringArray1.set(1, "token2FeatureStructures1Feature3Feature3_0_1");
+ stringArray1.set(2, "token2FeatureStructures1Feature3Feature3_0_2");
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature3(stringArray1);
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token2FeatureStructures1Feature3Feature3_1_0");
+ stringArray1.set(1, "token2FeatureStructures1Feature3Feature3_1_1");
+ stringArray1.set(2, "token2FeatureStructures1Feature3Feature3_1_2");
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature3(stringArray1);
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(1, featureStructure1);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token2FeatureStructures1Feature3Feature3_2_0");
+ stringArray1.set(1, "token2FeatureStructures1Feature3Feature3_2_1");
+ stringArray1.set(2, "token2FeatureStructures1Feature3Feature3_2_2");
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature3(stringArray1);
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation2.setFeatureStructures1(featureStructure1Array);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+
+ featureStructure1Array = new FSArray(cas, 3);
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token3FeatureStructures1Feature3Feature3_0_0");
+ stringArray1.set(1, "token3FeatureStructures1Feature3Feature3_0_1");
+ stringArray1.set(2, "token3FeatureStructures1Feature3Feature3_0_2");
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature3(stringArray1);
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(0, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token3FeatureStructures1Feature3Feature3_1_0");
+ stringArray1.set(1, "token3FeatureStructures1Feature3Feature3_1_1");
+ stringArray1.set(2, "token3FeatureStructures1Feature3Feature3_1_2");
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature3(stringArray1);
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(1, featureStructure1);
+
+ featureStructure1 = new FeatureStructure1(cas);
+
+ stringArray1 = new StringArray(cas, 3);
+ stringArray1.set(0, "token3FeatureStructures1Feature3Feature3_2_0");
+ stringArray1.set(1, "token3FeatureStructures1Feature3Feature3_2_1");
+ stringArray1.set(2, "token3FeatureStructures1Feature3Feature3_2_2");
+
+ featureStructure2 = new FeatureStructure2(cas);
+ featureStructure2.setFeature3(stringArray1);
+
+ featureStructure1.setFeature3(featureStructure2);
+ featureStructure1Array.set(2, featureStructure1);
+
+ annotation3.setFeatureStructures1(featureStructure1Array);
+ annotation3.addToIndexes();
+
+ List<String> featureNames = new ArrayList<String>();
+ featureNames.add("feature3");
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1", "featureStructures1.feature3",
+ featureNames, null);
+
+ Token nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_0_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_0_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_0_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_1_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_1_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_1_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_2_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_2_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1FeatureStructures1Feature3Feature3_2_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_0_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_0_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_0_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_1_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_1_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_1_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_2_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_2_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2FeatureStructures1Feature3Feature3_2_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_0_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_0_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_0_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_1_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_1_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_1_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_2_0", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_2_1", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3FeatureStructures1Feature3Feature3_2_2", new String(nextToken.termBuffer(),
+ 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+
+ }
+
+ public void testReset() throws Exception {
+ JCas cas = createCasWithText("token1 token2 token3");
+ Annotation1 annotation1 = new Annotation1(cas);
+ annotation1.setBegin(0);
+ annotation1.setEnd(6);
+ annotation1.addToIndexes();
+ Annotation1 annotation2 = new Annotation1(cas);
+ annotation2.setBegin(7);
+ annotation2.setEnd(13);
+ annotation2.addToIndexes();
+
+ Annotation1 annotation3 = new Annotation1(cas);
+ annotation3.setBegin(14);
+ annotation3.setEnd(20);
+ annotation3.addToIndexes();
+
+ AnnotationTokenStream annotationTokenStream =
+ new AnnotationTokenStream(cas, CAS.NAME_DEFAULT_SOFA,
+ "de.julielab.jules.types.test.Annotation1");
+ Token nextToken = annotationTokenStream.next();
+ nextToken = annotationTokenStream.next();
+ nextToken = annotationTokenStream.next();
+ annotationTokenStream.reset();
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = annotationTokenStream.next();
+ assertNotNull(nextToken);
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(14, nextToken.startOffset());
+ assertEquals(20, nextToken.endOffset());
+ }
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/AnnotationTokenStreamTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymTokenFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymTokenFilterTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymTokenFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymTokenFilterTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.uima.lucas.indexer.analysis.HypernymTokenFilter;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+public class HypernymTokenFilterTest extends TestCase {
+
+ public void testNext() throws IOException {
+ List<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 6, 11));
+ tokens.add(new Token("token3", 11, 17));
+ tokens.add(new Token("token4", 17, 23));
+
+ CollectionTokenStream tokenStream = new CollectionTokenStream(tokens);
+ Map<String, List<String>> hypernyms = new HashMap<String, List<String>>();
+ List<String> tokenHypernyms = new ArrayList<String>();
+ tokenHypernyms.add("token21");
+ tokenHypernyms.add("token22");
+ tokenHypernyms.add("token23");
+ hypernyms.put("token2", tokenHypernyms);
+
+ tokenHypernyms = new ArrayList<String>();
+ tokenHypernyms.add("token41");
+ tokenHypernyms.add("token42");
+ hypernyms.put("token4", tokenHypernyms);
+
+ HypernymTokenFilter tokenFilter = new HypernymTokenFilter(tokenStream, hypernyms);
+
+ Token nextToken = tokenFilter.next();
+ assertEquals("token1", nextToken.termText());
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token2", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token21", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token22", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token23", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token3", nextToken.termText());
+ assertEquals(11, nextToken.startOffset());
+ assertEquals(17, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token4", nextToken.termText());
+ assertEquals(17, nextToken.startOffset());
+ assertEquals(23, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token41", nextToken.termText());
+ assertEquals(17, nextToken.startOffset());
+ assertEquals(23, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token42", nextToken.termText());
+ assertEquals(17, nextToken.startOffset());
+ assertEquals(23, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ }
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymTokenFilterTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.analysis.PositionFilter;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+import org.junit.Test;
+
+public class PositionFilterTest {
+
+ @Test
+ public void testNext() throws Exception {
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token1", 7, 13));
+ tokens.add(new Token("token2", 14, 20));
+ tokens.add(new Token("token2", 21, 27));
+ tokens.add(new Token("token3", 28, 33));
+ tokens.add(new Token("token4", 34, 40));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ TokenFilter filter = new PositionFilter(tokenStream, PositionFilter.FIRST_POSITION);
+
+ Token nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNull(nextToken);
+
+ filter = new PositionFilter(tokenStream, PositionFilter.LAST_POSITION);
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNull(nextToken);
+ }
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.analysis.ReplaceFilter;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+public class ReplaceFilterTest extends TestCase {
+
+ public void testNext() throws Exception {
+ Map<String, String> mapping = new HashMap<String, String>();
+ mapping.put("token1", "replacement1");
+ mapping.put("token2", "replacement2");
+ mapping.put("token3", "replacement3");
+
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 7, 13));
+ tokens.add(new Token("token3", 14, 20));
+ tokens.add(new Token("token4", 21, 27));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ ReplaceFilter filter = new ReplaceFilter(tokenStream, mapping);
+
+ Token next = filter.next();
+ assertEquals("replacement1", new String(next.termBuffer(), 0, next.termLength()));
+ next = filter.next();
+ assertEquals("replacement2", new String(next.termBuffer(), 0, next.termLength()));
+ next = filter.next();
+ assertEquals("replacement3", new String(next.termBuffer(), 0, next.termLength()));
+ next = filter.next();
+ assertEquals("token4", new String(next.termBuffer(), 0, next.termLength()));
+
+ }
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.analysis.SplitterFilter;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+import org.junit.Test;
+
+public class SplitterFilterTest {
+
+ @Test
+ public void testNext() throws Exception {
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1 token2 token3", 0, 6));
+ tokens.add(new Token("token4 token5 token6", 7, 13));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ TokenFilter filter = new SplitterFilter(tokenStream, " ");
+
+ Token nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+ }
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+public class TokenStreamConcatenatorTest extends TestCase {
+
+ public void testNext() throws Exception {
+ Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
+ List<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 7, 13));
+ tokens.add(new Token("token3", 14, 20));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ tokenStreams.add(tokenStream);
+
+ tokens = new ArrayList<Token>();
+ tokens.add(new Token("token4", 21, 27));
+ tokens.add(new Token("token5", 28, 33));
+ tokens.add(new Token("token6", 34, 40));
+
+ tokenStream = new CollectionTokenStream(tokens);
+ tokenStreams.add(tokenStream);
+
+ TokenStreamConcatenator concatenator = new TokenStreamConcatenator(tokenStreams);
+
+ Token nextToken = concatenator.next();
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ concatenator.reset();
+ nextToken = concatenator.next();
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ }
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java?rev=780236&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java Sat May 30 11:12:58 2009
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.analysis.TokenStreamMerger;
+import org.apache.uima.lucas.indexer.test.util.DummyTokenStream;
+
+public class TokenStreamMergerTest extends TestCase {
+ private TokenStreamMerger merger;
+
+ @Override
+ protected void setUp() throws Exception {
+ List<TokenStream> streams = new ArrayList<TokenStream>();
+ streams.add(new DummyTokenStream("1111", 1, 4, 0));
+ streams.add(new DummyTokenStream("2222", 2, 2, 1));
+ streams.add(new DummyTokenStream("3333", 3, 1, 2));
+ merger = new TokenStreamMerger(streams);
+ }
+
+ public void testNext() throws IOException {
+
+ Token currentToken = merger.next();
+
+ assertEquals("1111", currentToken.termText());
+ assertEquals(0, currentToken.startOffset());
+ assertEquals(4, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("1111", currentToken.termText());
+ assertEquals(5, currentToken.startOffset());
+ assertEquals(9, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("2222", currentToken.termText());
+ assertEquals(5, currentToken.startOffset());
+ assertEquals(9, currentToken.endOffset());
+ assertEquals(0, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("1111", currentToken.termText());
+ assertEquals(10, currentToken.startOffset());
+ assertEquals(14, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("3333", currentToken.termText());
+ assertEquals(10, currentToken.startOffset());
+ assertEquals(14, currentToken.endOffset());
+ assertEquals(0, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("1111", currentToken.termText());
+ assertEquals(15, currentToken.startOffset());
+ assertEquals(19, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("2222", currentToken.termText());
+ assertEquals(15, currentToken.startOffset());
+ assertEquals(19, currentToken.endOffset());
+ assertEquals(0, currentToken.getPositionIncrement());
+ }
+
+}
Propchange: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain