You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by jo...@apache.org on 2009/07/24 10:58:55 UTC
svn commit: r797373 [6/8] - in /incubator/uima/sandbox/trunk/Lucas: ./
docbook/ docbook/LuceneCASConsumerUserGuide/
docbook/LuceneCASConsumerUserGuide/images/
src/main/java/org/apache/uima/lucas/consumer/
src/main/java/org/apache/uima/lucas/indexer/ sr...
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.classextension.EasyMock.*;
+import static org.junit.Assert.*;
+
+public class ConcatFilterFactoryTest {
+
+ private ConcatFilterFactory concatFilterFactory;
+ private TokenStream tokenStream;
+
+ @Before
+ public void setUp(){
+ tokenStream = createMock(TokenStream.class);
+ concatFilterFactory = new ConcatFilterFactory();
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws IOException{
+ Properties properties = new Properties();
+ properties.setProperty(ConcatFilterFactory.CONCAT_STRING_PARAMETER, "|");
+ ConcatFilter concatFilter = (ConcatFilter) concatFilterFactory.createTokenFilter(tokenStream, properties );
+ assertNotNull(concatFilter);
+ assertEquals("|", concatFilter.getConcatString());
+ }
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ConcatFilterTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.classextension.EasyMock.*;
+import static org.junit.Assert.*;
+
+public class ConcatFilterTest {
+
+ private ConcatFilter concatFilter;
+ private TokenStream tokenStream;
+ private Token token1;
+ private Token token2;
+ private Token token3;
+ private Token token;
+
+ @Before
+ public void setUp() throws IOException{
+ tokenStream = createMock(TokenStream.class);
+ concatFilter = new ConcatFilter(tokenStream, "|");
+ token1 = new Token("token1".toCharArray(), 0 , 6, 0, 5);
+ token2 = new Token("token2".toCharArray(), 0 , 6, 6, 11);
+ token3 = new Token("token3".toCharArray(), 0 , 6, 12, 17);
+ token = new Token();
+
+ setUpInputTokenStream();
+ }
+
+ @Test
+ public void testNext() throws IOException{
+ Token concatenatedToken = concatFilter.next(token);
+ verify(tokenStream);
+
+ assertEquals(token, concatenatedToken);
+ assertEquals(0, concatenatedToken.startOffset());
+ assertEquals(17, concatenatedToken.endOffset());
+ assertEquals("token1|token2|token3", concatenatedToken.term());
+ }
+
+ private void setUpInputTokenStream() throws IOException {
+ expect(tokenStream.next(token)).andReturn(token1);
+ expect(tokenStream.next(token)).andReturn(token2);
+ expect(tokenStream.next(token)).andReturn(token3);
+ expect(tokenStream.next(token)).andReturn(null);
+ replay(tokenStream);
+ }
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/DefaultFilterFactoryRegistryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/DefaultFilterFactoryRegistryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/DefaultFilterFactoryRegistryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/DefaultFilterFactoryRegistryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class DefaultFilterFactoryRegistryTest {
+
+ private DefaultFilterFactoryRegistry defaultFilterFactoryRegistry;
+
+ @Before
+ public void setUp() throws Exception {
+ this.defaultFilterFactoryRegistry = new DefaultFilterFactoryRegistry();
+ }
+
+ @Test
+ public void testGetDefaultFilterFactories() {
+ Map<String, TokenFilterFactory> registry = defaultFilterFactoryRegistry.getDefaultRegistry();
+ assertEquals(12, registry.size());
+
+ TokenFilterFactory filterFactory = registry.get(DefaultFilterFactoryRegistry.ADDITION_FILTER_FACTORY_NAME);
+ assertEquals(AdditionFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.HYPERNYM_FILTER_FACTORY_NAME);
+ assertEquals(HypernymFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.POSITION_FILTER_FACTORY_NAME);
+ assertEquals(PositionFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.REPLACE_FILTER_FACTORY_NAME);
+ assertEquals(ReplaceFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.SNOWBALL_FILTER_FACTORY_NAME);
+ assertEquals(SnowballFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.SPLITTER_FILTER_FACTORY_NAME);
+ assertEquals(SplitterFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.CONCAT_FILTER_FACTORY_NAME);
+ assertEquals(ConcatFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.STOPWORD_FILTER_FACTORY_NAME);
+ assertEquals(StopwordFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.UNIQUE_FILTER_FACTORY_NAME);
+ assertEquals(UniqueFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.UPPERCASE_FILTER_FACTORY_NAME);
+ assertEquals(UpperCaseFilterFactory.class, filterFactory.getClass());
+
+ filterFactory = registry.get(DefaultFilterFactoryRegistry.LOWERCASE_FILTER_FACTORY_NAME);
+ assertEquals(LowerCaseFilterFactory.class, filterFactory.getClass());
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.util.MultimapFileReader;
+import org.apache.uima.lucas.indexer.util.MultimapFileReaderFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+import static org.easymock.classextension.EasyMock.*;
+import static org.junit.Assert.*;
+
+public class HypernymFilterFactoryTest {
+
+ private static final String FILE_ENTRY = "token1=id1,id2,id3";
+ private static final String HYPERNYM_FILE = "src/test/resources/hypernyms.txt";
+ private HypernymFilterFactory hypernymFilterFactory;
+ private TokenStream tokenStream;
+ private MultimapFileReaderFactory multimapFileReaderFactory;
+ private MultimapFileReader multimapFileReader;
+ private Map<String, List<String>> hypernyms;
+
+ @Before
+ public void setUp(){
+ tokenStream = createMock(TokenStream.class);
+ hypernyms = new HashMap<String, List<String>>();
+ hypernyms.put("token1", Lists.newArrayList("id1", "id2", "id3"));
+
+ multimapFileReader = createMock(MultimapFileReader.class);
+ multimapFileReaderFactory = createMock(MultimapFileReaderFactory.class);
+ hypernymFilterFactory = new HypernymFilterFactory(multimapFileReaderFactory);
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(HypernymFilterFactory.FILE_PATH_PARAMETER, HYPERNYM_FILE);
+
+ expect(multimapFileReaderFactory.createMultimapFileReader(HYPERNYM_FILE)).andReturn(multimapFileReader);
+ expect(multimapFileReader.readMultimap()).andReturn(hypernyms);
+ replay(multimapFileReaderFactory);
+ replay(multimapFileReader);
+
+ HypernymFilter hypernymFilter = (HypernymFilter) hypernymFilterFactory.createTokenFilter(tokenStream, properties);
+ Map<String, Map<String, List<String>>> mappings = hypernymFilterFactory.getCachedMappings();
+ verify(multimapFileReaderFactory);
+ verify(multimapFileReader);
+
+ Map<String, List<String>> mapping = mappings.get(HYPERNYM_FILE);
+ assertEquals(hypernyms, mapping);
+ assertEquals(hypernyms, hypernymFilter.getHypernyms());
+
+ // test caching
+ reset(multimapFileReaderFactory);
+ reset(multimapFileReader);
+ replay(multimapFileReaderFactory);
+ replay(multimapFileReader);
+
+ hypernymFilter = (HypernymFilter) hypernymFilterFactory.createTokenFilter(tokenStream, properties);
+ verify(multimapFileReaderFactory);
+ verify(multimapFileReader);
+
+ assertEquals(hypernyms, hypernymFilter.getHypernyms());
+ }
+
+ public void testPreloadResources() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(HypernymFilterFactory.FILE_PATH_PARAMETER, HYPERNYM_FILE);
+
+ expect(multimapFileReaderFactory.createMultimapFileReader(FILE_ENTRY)).andReturn(multimapFileReader);
+ expect(multimapFileReader.readMultimap()).andReturn(hypernyms);
+ hypernymFilterFactory.preloadResources(properties);
+
+ Map<String, Map<String, List<String>>> mappings = hypernymFilterFactory.getCachedMappings();
+ verify(multimapFileReaderFactory);
+ verify(multimapFileReader);
+
+ Map<String, List<String>> mapping = mappings.get(HYPERNYM_FILE);
+ assertEquals(hypernyms, mapping);
+
+ // test caching
+ reset(multimapFileReaderFactory);
+ reset(multimapFileReader);
+ replay(multimapFileReaderFactory);
+ replay(multimapFileReader);
+
+ hypernymFilterFactory.preloadResources(properties);
+ mappings = hypernymFilterFactory.getCachedMappings();
+
+ verify(multimapFileReaderFactory);
+ verify(multimapFileReader);
+
+ mapping = mappings.get(HYPERNYM_FILE);
+ assertEquals(hypernyms, mapping);
+ }
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/HypernymFilterTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.uima.lucas.indexer.analysis.HypernymFilter;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+public class HypernymFilterTest extends TestCase {
+
+ public void testNext() throws IOException{
+ List<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 6, 11));
+ tokens.add(new Token("token3", 11, 17));
+ tokens.add(new Token("token4", 17, 23));
+
+ CollectionTokenStream tokenStream = new CollectionTokenStream(tokens);
+ Map<String, List<String>> hypernyms = new HashMap<String, List<String>>();
+ List<String> tokenHypernyms = new ArrayList<String>();
+ tokenHypernyms.add("token21");
+ tokenHypernyms.add("token22");
+ tokenHypernyms.add("token23");
+ hypernyms.put("token2", tokenHypernyms);
+
+ tokenHypernyms = new ArrayList<String>();
+ tokenHypernyms.add("token41");
+ tokenHypernyms.add("token42");
+ hypernyms.put("token4", tokenHypernyms);
+
+ HypernymFilter tokenFilter = new HypernymFilter(tokenStream, hypernyms);
+
+ Token nextToken = tokenFilter.next();
+ assertEquals("token1", nextToken.termText());
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token2", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token21", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token22", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token23", nextToken.termText());
+ assertEquals(6, nextToken.startOffset());
+ assertEquals(11, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token3", nextToken.termText());
+ assertEquals(11, nextToken.startOffset());
+ assertEquals(17, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token4", nextToken.termText());
+ assertEquals(17, nextToken.startOffset());
+ assertEquals(23, nextToken.endOffset());
+ assertEquals(1, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token41", nextToken.termText());
+ assertEquals(17, nextToken.startOffset());
+ assertEquals(23, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ nextToken = tokenFilter.next();
+ assertEquals("token42", nextToken.termText());
+ assertEquals(17, nextToken.startOffset());
+ assertEquals(23, nextToken.endOffset());
+ assertEquals(0, nextToken.getPositionIncrement());
+
+ }
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/LowerCaseFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/LowerCaseFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/LowerCaseFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/LowerCaseFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+import static org.easymock.classextension.EasyMock.*;
+
+public class LowerCaseFilterFactoryTest {
+
+ private LowerCaseFilterFactory lowercaseFilterFactory;
+ private TokenStream tokenStream;
+
+ @Before
+ public void setUp() throws Exception {
+ this.lowercaseFilterFactory = new LowerCaseFilterFactory();
+ this.tokenStream = createMock(TokenStream.class);
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws IOException {
+ LowerCaseFilter lowercaseFilter = (LowerCaseFilter) lowercaseFilterFactory.createTokenFilter(tokenStream, null);
+ assertNotNull(lowercaseFilter);
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+
+public class PositionFilterFactoryTest {
+
+ private static final String FIRST = "first";
+ private TokenFilterFactory positionFilterFactory;
+ private TokenStream tokenStream;
+
+ @Before
+ public void setUp(){
+ positionFilterFactory = new PositionFilterFactory();
+ tokenStream = createMock(TokenStream.class);
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws IOException{
+ Properties properties = new Properties();
+ properties.setProperty(PositionFilterFactory.POSITION_PARAMETER, PositionFilterFactory.FIRST_POSITION_PARAMETER_VALUE);
+
+ PositionFilter positionTokenFilter = (PositionFilter) positionFilterFactory.createTokenFilter(tokenStream, properties);
+ assertEquals(PositionFilter.FIRST_POSITION, positionTokenFilter.getPosition());
+
+ properties = new Properties();
+ properties.setProperty(PositionFilterFactory.POSITION_PARAMETER, PositionFilterFactory.LAST_POSITION_PARAMETER_VALUE);
+
+ positionTokenFilter = (PositionFilter) positionFilterFactory.createTokenFilter(tokenStream, properties);
+ assertEquals(PositionFilter.LAST_POSITION, positionTokenFilter.getPosition());
+ }
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/PositionFilterTest.java Fri Jul 24 08:58:52 2009
@@ -19,9 +19,7 @@
package org.apache.uima.lucas.indexer.analysis;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.Collection;
@@ -33,34 +31,37 @@
import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
import org.junit.Test;
+
+
public class PositionFilterTest {
- @Test
- public void testNext() throws Exception {
- Collection<Token> tokens = new ArrayList<Token>();
- tokens.add(new Token("token1", 0, 6));
- tokens.add(new Token("token1", 7, 13));
- tokens.add(new Token("token2", 14, 20));
- tokens.add(new Token("token2", 21, 27));
- tokens.add(new Token("token3", 28, 33));
- tokens.add(new Token("token4", 34, 40));
-
- TokenStream tokenStream = new CollectionTokenStream(tokens);
- TokenFilter filter = new PositionFilter(tokenStream, PositionFilter.FIRST_POSITION);
-
- Token nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
-
- nextToken = filter.next();
- assertNull(nextToken);
-
- filter = new PositionFilter(tokenStream, PositionFilter.LAST_POSITION);
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
-
- nextToken = filter.next();
- assertNull(nextToken);
- }
+ @Test
+ public void testNext() throws Exception{
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token1", 7, 13));
+ tokens.add(new Token("token2", 14, 20));
+ tokens.add(new Token("token2", 21, 27));
+ tokens.add(new Token("token3", 28, 33));
+ tokens.add(new Token("token4", 34, 40));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ TokenFilter filter = new PositionFilter(tokenStream, PositionFilter.FIRST_POSITION);
+
+ Token nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNull(nextToken);
+
+ filter = new PositionFilter(tokenStream, PositionFilter.LAST_POSITION);
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNull(nextToken);
+ }
+
}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.easymock.classextension.EasyMock.replay;
+import static org.easymock.classextension.EasyMock.reset;
+import static org.easymock.classextension.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.util.MapFileReader;
+import org.apache.uima.lucas.indexer.util.MapFileReaderFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ReplaceFilterFactoryTest {
+
+ private static final String TEST_FILE_1 = "src/test/resources/ReplaceFilterFactoryTest1.txt";
+ private ReplaceFilterFactory replaceFilterFactory;
+ private MapFileReaderFactory mapFileReaderFactory;
+ private MapFileReader mapFileReader;
+ private TokenStream tokenStream;
+ private Map<String, String> mapping;
+
+ @Before
+ public void setUp(){
+ mapFileReaderFactory = createMock(MapFileReaderFactory.class);
+ mapFileReader= createMock(MapFileReader.class);
+ replaceFilterFactory = new ReplaceFilterFactory(mapFileReaderFactory);
+ tokenStream = createMock(TokenStream.class);
+ mapping = new HashMap<String, String>();
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(ReplaceFilterFactory.FILE_PATH_PARAMETER, TEST_FILE_1);
+
+ expect(mapFileReaderFactory.createMapFileReader(TEST_FILE_1)).andReturn(mapFileReader);
+ expect(mapFileReader.readMap()).andReturn(mapping);
+ replay(mapFileReaderFactory);
+ replay(mapFileReader);
+
+ TokenFilter tokenFilter = replaceFilterFactory.createTokenFilter(tokenStream, properties);
+ assertEquals(ReplaceFilter.class, tokenFilter.getClass());
+ ReplaceFilter replaceFilter = (ReplaceFilter) tokenFilter;
+ assertEquals(mapping, replaceFilter.getMapping());
+ verify(mapFileReaderFactory);
+ verify(mapFileReader);
+
+ reset(mapFileReaderFactory);
+ reset(mapFileReader);
+ replay(mapFileReaderFactory);
+ replay(mapFileReader);
+
+ // test caching (no calls of buffered reader)
+ tokenFilter = replaceFilterFactory.createTokenFilter(tokenStream, properties);
+ verify(mapFileReaderFactory);
+ verify(mapFileReader);
+
+ replaceFilter = (ReplaceFilter) tokenFilter;
+ assertEquals(mapping, replaceFilter.getMapping());
+ }
+
+ @Test
+ public void testPreloadResources() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(ReplaceFilterFactory.FILE_PATH_PARAMETER, TEST_FILE_1);
+
+ expect(mapFileReaderFactory.createMapFileReader(TEST_FILE_1)).andReturn(mapFileReader);
+ expect(mapFileReader.readMap()).andReturn(mapping);
+ replay(mapFileReaderFactory);
+ replay(mapFileReader);
+
+ replaceFilterFactory.preloadResources(properties);
+ Map<String, String> cachedMapping = replaceFilterFactory.getCachedMappings().get(TEST_FILE_1);
+ assertEquals(mapping, cachedMapping);
+ verify(mapFileReaderFactory);
+ verify(mapFileReader);
+
+ reset(mapFileReaderFactory);
+ reset(mapFileReader);
+ replay(mapFileReaderFactory);
+ replay(mapFileReader);
+
+ // test caching (no calls of buffered reader)
+ replaceFilterFactory.preloadResources(properties);
+ cachedMapping = replaceFilterFactory.getCachedMappings().get(TEST_FILE_1);
+ assertEquals(mapping, cachedMapping);
+ verify(mapFileReaderFactory);
+ verify(mapFileReader);
+ }
+
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/ReplaceFilterTest.java Fri Jul 24 08:58:52 2009
@@ -31,31 +31,34 @@
import org.apache.uima.lucas.indexer.analysis.ReplaceFilter;
import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
-public class ReplaceFilterTest extends TestCase {
- public void testNext() throws Exception {
- Map<String, String> mapping = new HashMap<String, String>();
- mapping.put("token1", "replacement1");
- mapping.put("token2", "replacement2");
- mapping.put("token3", "replacement3");
- Collection<Token> tokens = new ArrayList<Token>();
- tokens.add(new Token("token1", 0, 6));
- tokens.add(new Token("token2", 7, 13));
- tokens.add(new Token("token3", 14, 20));
- tokens.add(new Token("token4", 21, 27));
+public class ReplaceFilterTest extends TestCase{
- TokenStream tokenStream = new CollectionTokenStream(tokens);
- ReplaceFilter filter = new ReplaceFilter(tokenStream, mapping);
-
- Token next = filter.next();
- assertEquals("replacement1", new String(next.termBuffer(), 0, next.termLength()));
- next = filter.next();
- assertEquals("replacement2", new String(next.termBuffer(), 0, next.termLength()));
- next = filter.next();
- assertEquals("replacement3", new String(next.termBuffer(), 0, next.termLength()));
- next = filter.next();
- assertEquals("token4", new String(next.termBuffer(), 0, next.termLength()));
-
- }
+ public void testNext() throws Exception{
+ Map<String, String> mapping = new HashMap<String, String>();
+ mapping.put("token1", "replacement1");
+ mapping.put("token2", "replacement2");
+ mapping.put("token3", "replacement3");
+
+
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 7, 13));
+ tokens.add(new Token("token3", 14, 20));
+ tokens.add(new Token("token4", 21, 27));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ ReplaceFilter filter = new ReplaceFilter(tokenStream, mapping);
+
+ Token next = filter.next();
+ assertEquals("replacement1", new String(next.termBuffer(), 0, next.termLength()));
+ next = filter.next();
+ assertEquals("replacement2", new String(next.termBuffer(), 0, next.termLength()));
+ next = filter.next();
+ assertEquals("replacement3", new String(next.termBuffer(), 0, next.termLength()));
+ next = filter.next();
+ assertEquals("token4", new String(next.termBuffer(), 0, next.termLength()));
+
+ }
}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SnowballFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SnowballFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SnowballFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SnowballFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.snowball.SnowballFilter;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SnowballFilterFactoryTest {
+
+ private static final String PORTER = "Porter";
+ private TokenStream tokenStream;
+ private SnowballFilterFactory snowballFilterFactory;
+
+ @Before
+ public void setUp(){
+ tokenStream = createMock(TokenStream.class);
+ snowballFilterFactory = new SnowballFilterFactory();
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(SnowballFilterFactory.STEMMER_NAME_PARAMETER, PORTER);
+
+ SnowballFilter snowballFilter = (SnowballFilter) snowballFilterFactory.createTokenFilter(tokenStream, properties );
+ assertNotNull(snowballFilter);
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.util.Properties;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+import static org.easymock.classextension.EasyMock.*;
+import static org.junit.Assert.*;
+
+public class SplitterFilterFactoryTest {
+
+ private SplitterFilterFactory splitterFilterFactory;
+ private TokenStream tokenStream;
+
+ @Before
+ public void setUp(){
+ tokenStream = createMock(TokenStream.class);
+ splitterFilterFactory = new SplitterFilterFactory();
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(SplitterFilterFactory.SPLIT_STRING_PARAMETER, ",");
+ SplitterFilter splitterFilter = (SplitterFilter) splitterFilterFactory.createTokenFilter(tokenStream, properties );
+ assertEquals("," , splitterFilter.getSplitString());
+ }
+
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/SplitterFilterTest.java Fri Jul 24 08:58:52 2009
@@ -34,49 +34,49 @@
public class SplitterFilterTest {
- @Test
- public void testNext() throws Exception {
- Collection<Token> tokens = new ArrayList<Token>();
- tokens.add(new Token("token1 token2 token3", 0, 6));
- tokens.add(new Token("token4 token5 token6", 7, 13));
-
- TokenStream tokenStream = new CollectionTokenStream(tokens);
- TokenFilter filter = new SplitterFilter(tokenStream, " ");
-
- Token nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertEquals(0, nextToken.startOffset());
- assertEquals(6, nextToken.endOffset());
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertEquals(0, nextToken.startOffset());
- assertEquals(6, nextToken.endOffset());
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertEquals(0, nextToken.startOffset());
- assertEquals(6, nextToken.endOffset());
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertEquals(7, nextToken.startOffset());
- assertEquals(13, nextToken.endOffset());
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertEquals(7, nextToken.startOffset());
- assertEquals(13, nextToken.endOffset());
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertEquals(7, nextToken.startOffset());
- assertEquals(13, nextToken.endOffset());
- }
+ @Test
+ public void testNext() throws Exception{
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1 token2 token3", 0, 6));
+ tokens.add(new Token("token4 token5 token6", 7, 13));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ TokenFilter filter = new SplitterFilter(tokenStream, " ");
+
+ Token nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(0, nextToken.startOffset());
+ assertEquals(6, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ assertEquals(7, nextToken.startOffset());
+ assertEquals(13, nextToken.endOffset());
+ }
}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/StopwordFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/StopwordFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/StopwordFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/StopwordFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.easymock.classextension.EasyMock.replay;
+import static org.easymock.classextension.EasyMock.reset;
+import static org.easymock.classextension.EasyMock.verify;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.Properties;
+
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.util.PlainFileReader;
+import org.apache.uima.lucas.indexer.util.PlainFileReaderFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StopwordFilterFactoryTest {
+ private static final String TEST_FILE_1 = "src/test/resources/ReplaceFilterFactoryTest1.txt";
+ private StopwordFilterFactory stopwordFilterFactory;
+ private PlainFileReaderFactory plainFileReaderFactory;
+ private PlainFileReader plainFileReader;
+ private TokenStream tokenStream;
+ private String[] lines;
+
+ @Before
+ public void setUp(){
+ plainFileReaderFactory = createMock(PlainFileReaderFactory.class);
+ plainFileReader= createMock(PlainFileReader.class);
+ stopwordFilterFactory = new StopwordFilterFactory(plainFileReaderFactory);
+ tokenStream = createMock(TokenStream.class);
+ lines = new String[]{};
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(StopwordFilterFactory.FILE_PATH_PARAMETER, TEST_FILE_1);
+
+ expect(plainFileReaderFactory.createPlainFileReader(TEST_FILE_1)).andReturn(plainFileReader);
+ expect(plainFileReader.readLines()).andReturn(lines);
+ replay(plainFileReaderFactory);
+ replay(plainFileReader);
+
+ StopFilter stopFilter = (StopFilter) stopwordFilterFactory.createTokenFilter(tokenStream, properties);
+ assertNotNull(stopFilter);
+ verify(plainFileReaderFactory);
+ verify(plainFileReader);
+
+ reset(plainFileReaderFactory);
+ reset(plainFileReader);
+ replay(plainFileReaderFactory);
+ replay(plainFileReader);
+
+ // test caching (no calls of buffered reader)
+ stopFilter = (StopFilter) stopwordFilterFactory.createTokenFilter(tokenStream, properties);
+ verify(plainFileReaderFactory);
+ verify(plainFileReader);
+ }
+
+ @Test
+ public void testPreloadResources() throws Exception{
+ Properties properties = new Properties();
+ properties.setProperty(ReplaceFilterFactory.FILE_PATH_PARAMETER, TEST_FILE_1);
+
+ expect(plainFileReaderFactory.createPlainFileReader(TEST_FILE_1)).andReturn(plainFileReader);
+ expect(plainFileReader.readLines()).andReturn(lines);
+ replay(plainFileReaderFactory);
+ replay(plainFileReader);
+
+ stopwordFilterFactory.preloadResources(properties);
+ verify(plainFileReaderFactory);
+ verify(plainFileReader);
+
+ reset(plainFileReaderFactory);
+ reset(plainFileReader);
+ replay(plainFileReaderFactory);
+ replay(plainFileReader);
+
+ // test caching (no calls of buffered reader)
+ stopwordFilterFactory.preloadResources(properties);
+ verify(plainFileReaderFactory);
+ verify(plainFileReader);
+ }
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamConcatenatorTest.java Fri Jul 24 08:58:52 2009
@@ -23,60 +23,61 @@
import java.util.Collection;
import java.util.List;
-import junit.framework.TestCase;
-
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.uima.lucas.indexer.analysis.TokenStreamConcatenator;
import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+import junit.framework.TestCase;
+
public class TokenStreamConcatenatorTest extends TestCase {
- public void testNext() throws Exception {
- Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
- List<Token> tokens = new ArrayList<Token>();
- tokens.add(new Token("token1", 0, 6));
- tokens.add(new Token("token2", 7, 13));
- tokens.add(new Token("token3", 14, 20));
-
- TokenStream tokenStream = new CollectionTokenStream(tokens);
- tokenStreams.add(tokenStream);
-
- tokens = new ArrayList<Token>();
- tokens.add(new Token("token4", 21, 27));
- tokens.add(new Token("token5", 28, 33));
- tokens.add(new Token("token6", 34, 40));
-
- tokenStream = new CollectionTokenStream(tokens);
- tokenStreams.add(tokenStream);
-
- TokenStreamConcatenator concatenator = new TokenStreamConcatenator(tokenStreams);
-
- Token nextToken = concatenator.next();
- assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
-
- concatenator.reset();
- nextToken = concatenator.next();
- assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- nextToken = concatenator.next();
- assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- }
+ public void testNext() throws Exception{
+ Collection<TokenStream> tokenStreams = new ArrayList<TokenStream>();
+ List<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 7, 13));
+ tokens.add(new Token("token3", 14, 20));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens );
+ tokenStreams.add(tokenStream);
+
+ tokens = new ArrayList<Token>();
+ tokens.add(new Token("token4", 21, 27));
+ tokens.add(new Token("token5", 28, 33));
+ tokens.add(new Token("token6", 34, 40));
+
+ tokenStream = new CollectionTokenStream(tokens );
+ tokenStreams.add(tokenStream);
+
+ TokenStreamConcatenator concatenator = new TokenStreamConcatenator(tokenStreams);
+
+ Token nextToken = concatenator.next();
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ concatenator.reset();
+ nextToken = concatenator.next();
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token5", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ nextToken = concatenator.next();
+ assertEquals("token6", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ }
}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/TokenStreamMergerTest.java Fri Jul 24 08:58:52 2009
@@ -23,6 +23,7 @@
import java.util.ArrayList;
import java.util.List;
+
import junit.framework.TestCase;
import org.apache.lucene.analysis.Token;
@@ -31,61 +32,60 @@
import org.apache.uima.lucas.indexer.test.util.DummyTokenStream;
public class TokenStreamMergerTest extends TestCase {
- private TokenStreamMerger merger;
-
- @Override
- protected void setUp() throws Exception {
- List<TokenStream> streams = new ArrayList<TokenStream>();
- streams.add(new DummyTokenStream("1111", 1, 4, 0));
- streams.add(new DummyTokenStream("2222", 2, 2, 1));
- streams.add(new DummyTokenStream("3333", 3, 1, 2));
- merger = new TokenStreamMerger(streams);
- }
-
- public void testNext() throws IOException {
-
- Token currentToken = merger.next();
-
- assertEquals("1111", currentToken.termText());
- assertEquals(0, currentToken.startOffset());
- assertEquals(4, currentToken.endOffset());
- assertEquals(1, currentToken.getPositionIncrement());
-
- currentToken = merger.next();
- assertEquals("1111", currentToken.termText());
- assertEquals(5, currentToken.startOffset());
- assertEquals(9, currentToken.endOffset());
- assertEquals(1, currentToken.getPositionIncrement());
-
- currentToken = merger.next();
- assertEquals("2222", currentToken.termText());
- assertEquals(5, currentToken.startOffset());
- assertEquals(9, currentToken.endOffset());
- assertEquals(0, currentToken.getPositionIncrement());
-
- currentToken = merger.next();
- assertEquals("1111", currentToken.termText());
- assertEquals(10, currentToken.startOffset());
- assertEquals(14, currentToken.endOffset());
- assertEquals(1, currentToken.getPositionIncrement());
-
- currentToken = merger.next();
- assertEquals("3333", currentToken.termText());
- assertEquals(10, currentToken.startOffset());
- assertEquals(14, currentToken.endOffset());
- assertEquals(0, currentToken.getPositionIncrement());
-
- currentToken = merger.next();
- assertEquals("1111", currentToken.termText());
- assertEquals(15, currentToken.startOffset());
- assertEquals(19, currentToken.endOffset());
- assertEquals(1, currentToken.getPositionIncrement());
-
- currentToken = merger.next();
- assertEquals("2222", currentToken.termText());
- assertEquals(15, currentToken.startOffset());
- assertEquals(19, currentToken.endOffset());
- assertEquals(0, currentToken.getPositionIncrement());
- }
+ private TokenStreamMerger merger;
+ @Override
+ protected void setUp() throws Exception {
+ List<TokenStream> streams = new ArrayList<TokenStream>();
+ streams.add(new DummyTokenStream("1111", 1, 4, 0));
+ streams.add(new DummyTokenStream("2222", 2, 2, 1));
+ streams.add(new DummyTokenStream("3333", 3, 1, 2));
+ merger = new TokenStreamMerger(streams);
+ }
+
+ public void testNext() throws IOException {
+
+ Token currentToken = merger.next();
+
+ assertEquals("1111", currentToken.termText());
+ assertEquals(0, currentToken.startOffset());
+ assertEquals(4, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("1111", currentToken.termText());
+ assertEquals(5, currentToken.startOffset());
+ assertEquals(9, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("2222", currentToken.termText());
+ assertEquals(5, currentToken.startOffset());
+ assertEquals(9, currentToken.endOffset());
+ assertEquals(0, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("1111", currentToken.termText());
+ assertEquals(10, currentToken.startOffset());
+ assertEquals(14, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("3333", currentToken.termText());
+ assertEquals(10, currentToken.startOffset());
+ assertEquals(14, currentToken.endOffset());
+ assertEquals(0, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("1111", currentToken.termText());
+ assertEquals(15, currentToken.startOffset());
+ assertEquals(19, currentToken.endOffset());
+ assertEquals(1, currentToken.getPositionIncrement());
+
+ currentToken = merger.next();
+ assertEquals("2222", currentToken.termText());
+ assertEquals(15, currentToken.startOffset());
+ assertEquals(19, currentToken.endOffset());
+ assertEquals(0, currentToken.getPositionIncrement());
+ }
}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.junit.Assert.*;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+
+public class UniqueFilterFactoryTest {
+
+ private TokenStream tokenStream;
+ private UniqueFilterFactory uniqueFilterFactory;
+
+ @Before
+ public void setUp(){
+ tokenStream = createMock(TokenStream.class);
+ uniqueFilterFactory = new UniqueFilterFactory();
+ }
+
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ UniqueFilter uniqueFilter = (UniqueFilter) uniqueFilterFactory.createTokenFilter(tokenStream, null);
+ assertNotNull(uniqueFilter);
+ }
+
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterTest.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterTest.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UniqueFilterTest.java Fri Jul 24 08:58:52 2009
@@ -22,44 +22,45 @@
import java.util.ArrayList;
import java.util.Collection;
-import junit.framework.TestCase;
-
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.uima.lucas.indexer.analysis.UniqueFilter;
import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+import junit.framework.TestCase;
+
public class UniqueFilterTest extends TestCase {
- public void testUniqueFilter() throws Exception {
- Collection<Token> tokens = new ArrayList<Token>();
- tokens.add(new Token("token1", 0, 6));
- tokens.add(new Token("token1", 7, 13));
- tokens.add(new Token("token2", 14, 20));
- tokens.add(new Token("token2", 21, 27));
- tokens.add(new Token("token3", 28, 33));
- tokens.add(new Token("token4", 34, 40));
-
- TokenStream tokenStream = new CollectionTokenStream(tokens);
- TokenFilter filter = new UniqueFilter(tokenStream);
-
- Token nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
-
- nextToken = filter.next();
- assertNotNull(nextToken);
- assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+ public void testUniqueFilter() throws Exception{
+ Collection<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token1", 7, 13));
+ tokens.add(new Token("token2", 14, 20));
+ tokens.add(new Token("token2", 21, 27));
+ tokens.add(new Token("token3", 28, 33));
+ tokens.add(new Token("token4", 34, 40));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens);
+ TokenFilter filter = new UniqueFilter(tokenStream);
+
+ Token nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token1", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token2", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token3", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
+
+ nextToken = filter.next();
+ assertNotNull(nextToken);
+ assertEquals("token4", new String(nextToken.termBuffer(), 0, nextToken.termLength()));
- assertNull(filter.next());
- }
+ assertNull(filter.next());
+ }
}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterFactoryTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterFactoryTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterFactoryTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterFactoryTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.junit.Before;
+import org.junit.Test;
+
+public class UppercaseFilterFactoryTest {
+
+ private TokenStream tokenStream;
+ private UpperCaseFilterFactory upperCaseFilterFactory;
+
+ @Before
+ public void setUp(){
+ tokenStream = createMock(TokenStream.class);
+ upperCaseFilterFactory = new UpperCaseFilterFactory();
+ }
+ @Test
+ public void testCreateTokenFilter() throws Exception{
+ UpperCaseFilter upperCaseFilter = (UpperCaseFilter) upperCaseFilterFactory.createTokenFilter(tokenStream, null);
+ assertNotNull(upperCaseFilter);
+ }
+
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/analysis/UppercaseFilterTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.analysis;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.uima.lucas.indexer.analysis.UpperCaseFilter;
+import org.apache.uima.lucas.indexer.test.util.CollectionTokenStream;
+
+
+public class UppercaseFilterTest extends TestCase {
+
+ public void testNext() throws Exception{
+ List<Token> tokens = new ArrayList<Token>();
+ tokens.add(new Token("token1", 0, 6));
+ tokens.add(new Token("token2", 7, 13));
+ tokens.add(new Token("token3", 14, 20));
+
+ TokenStream tokenStream = new CollectionTokenStream(tokens );
+
+ UpperCaseFilter upperCaseTokenFilter = new UpperCaseFilter(tokenStream);
+
+ Token next = upperCaseTokenFilter.next();
+
+ String nextString = new String(next.termBuffer(), 0, next.termLength());
+ assertEquals("TOKEN1", nextString);
+
+ next = upperCaseTokenFilter.next();
+ nextString = new String(next.termBuffer(), 0, next.termLength());
+ assertEquals("TOKEN2", nextString);
+
+ next = upperCaseTokenFilter.next();
+ nextString = new String(next.termBuffer(), 0, next.termLength());
+ assertEquals("TOKEN3", nextString);
+
+ }
+}
Added: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/mapping/MappingFileReaderTest.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/mapping/MappingFileReaderTest.java?rev=797373&view=auto
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/mapping/MappingFileReaderTest.java (added)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/mapping/MappingFileReaderTest.java Fri Jul 24 08:58:52 2009
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.lucas.indexer.mapping;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.xml.parsers.SAXParserFactory;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class MappingFileReaderTest {
+
+ private static final String FACTORY_NAME = "myFactory";
+ private static final String TESTFACTORY = "testfactory";
+ private static final String NUMBER_FORMAT = "##";
+ private static final String FEATURE_NAME = "featureInteger";
+ private static final String TOKENIZER = "cas";
+ private static final String VALUE_DELIMITER_STRING = "aValueDelimiterString";
+ private static final String FEATURE_PATH = "aFeaturePath";
+ private static final String SOFA = "aSofa";
+ private static final String ANNOTATION1_TYPE = "de.julielab.jules.types.test.Annotation1";
+ private static final String VALUE = "value";
+ private static final String KEY = "key";
+ private static final String TESTCLASS = "testfilter";
+ private static final String NO = "no";
+ private static final String YES = "yes";
+ private static final String FIELD_NAME = "annotation1";
+ private static final String MAPPING_FILE = "src/test/resources/MappingFileReaderTest.xml";
+ private MappingFileReader mappingFileReader;
+
+ @Before
+ public void setUp() throws Exception{
+ Map<String, ElementMapper<?>> elementMappers = new HashMap<String, ElementMapper<?>>();
+ elementMappers.put(MappingFileReader.ANNOTATION, new AnnotationMapper());
+ elementMappers.put(MappingFileReader.FILTER, new FilterMapper());
+ elementMappers.put(MappingFileReader.FIELD, new FieldMapper());
+ elementMappers.put(MappingFileReader.FEATURE, new FeatureMapper());
+
+ mappingFileReader = new MappingFileReader(SAXParserFactory.newInstance().newSAXParser(), elementMappers);
+ }
+
+ @Test
+ public void testCreateFieldDescriptions() throws IOException{
+ Collection<FieldDescription> fieldDescriptions = mappingFileReader.readFieldDescriptionsFromFile(new File(MAPPING_FILE));
+ assertEquals(1, fieldDescriptions.size());
+
+ FieldDescription fieldDescription = fieldDescriptions.iterator().next();
+ assertEquals(FIELD_NAME, fieldDescription.getName());
+ assertEquals(YES, fieldDescription.getIndex());
+ assertEquals(NO, fieldDescription.getTermVector());
+ assertEquals(YES, fieldDescription.getStored());
+ assertEquals(true, fieldDescription.getMerge());
+ assertEquals(6, fieldDescription.getLineNumber());
+ assertEquals(16, fieldDescription.getColumnNumber());
+
+ Collection<FilterDescription> filterDescriptions = fieldDescription.getFilterDescriptions();
+ assertEquals(1, filterDescriptions.size());
+ FilterDescription filterDescription = filterDescriptions.iterator().next();
+ assertEquals(TESTCLASS, filterDescription.getClassName());
+ assertEquals(TESTFACTORY, filterDescription.getFactoryClassName());
+ assertEquals(FACTORY_NAME, filterDescription.getName());
+ assertTrue(filterDescription.isReuseFactory());
+ assertEquals(9, filterDescription.getLineNumber());
+ assertEquals(56, filterDescription.getColumnNumber());
+
+ Properties properties = filterDescription.getProperties();
+ assertEquals(VALUE, properties.getProperty(KEY));
+
+ Collection<AnnotationDescription> annotationDescriptions = fieldDescription.getAnnotationDescriptions();
+ assertEquals(1, annotationDescriptions.size());
+ Iterator<AnnotationDescription> annotationDescriptionIterator = annotationDescriptions.iterator();
+
+ AnnotationDescription annotationDescription = annotationDescriptionIterator.next();
+ assertEquals(ANNOTATION1_TYPE, annotationDescription.getType());
+ assertEquals(SOFA, annotationDescription.getSofa());
+ assertEquals(FEATURE_PATH, annotationDescription.getFeaturePath());
+ assertEquals(VALUE_DELIMITER_STRING, annotationDescription.getFeatureValueDelimiterString());
+ assertEquals(TOKENIZER, annotationDescription.getTokenizer());
+ assertEquals(13, annotationDescription.getLineNumber());
+ assertEquals(113, annotationDescription.getColumnNumber());
+
+ filterDescriptions = annotationDescription.getFilterDescriptions();
+ assertEquals(1, filterDescriptions.size());
+ filterDescription = filterDescriptions.iterator().next();
+ assertEquals(TESTCLASS, filterDescription.getClassName());
+ assertEquals(TESTFACTORY, filterDescription.getFactoryClassName());
+ assertEquals(FACTORY_NAME, filterDescription.getName());
+ assertTrue(filterDescription.isReuseFactory());
+ assertEquals(16, filterDescription.getLineNumber());
+ assertEquals(58, filterDescription.getColumnNumber());
+
+ properties = filterDescription.getProperties();
+ assertEquals(VALUE, properties.getProperty(KEY));
+
+ Collection<FeatureDescription> featureDescriptions = annotationDescription.getFeatureDescriptions();
+ assertEquals(1, featureDescriptions.size());
+ FeatureDescription featureDescription = featureDescriptions.iterator().next();
+ assertEquals(FEATURE_NAME, featureDescription.getFeatureName());
+ assertEquals(NUMBER_FORMAT, featureDescription.getNumberFormat());
+ assertEquals(19, featureDescription.getLineNumber());
+ assertEquals(57, featureDescription.getColumnNumber());
+ }
+}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/CollectionTokenStream.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/CollectionTokenStream.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/CollectionTokenStream.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/CollectionTokenStream.java Fri Jul 24 08:58:52 2009
@@ -26,29 +26,30 @@
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-public class CollectionTokenStream extends TokenStream {
+public class CollectionTokenStream extends TokenStream{
- private Iterator<Token> tokenIterator;
-
- private Collection<Token> tokens;
-
- public CollectionTokenStream(Collection<Token> tokens) {
- super();
- this.tokenIterator = tokens.iterator();
- this.tokens = tokens;
- }
-
- @Override
- public Token next() throws IOException {
- if (tokenIterator.hasNext())
- return tokenIterator.next();
- else
- return null;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- this.tokenIterator = tokens.iterator();
- }
+ private Iterator<Token> tokenIterator;
+ private Collection<Token> tokens;
+
+
+ public CollectionTokenStream(Collection<Token> tokens) {
+ super();
+ this.tokenIterator = tokens.iterator();
+ this.tokens = tokens;
+ }
+
+
+ @Override
+ public Token next() throws IOException {
+ if( tokenIterator.hasNext() )
+ return tokenIterator.next();
+ else
+ return null;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ this.tokenIterator = tokens.iterator();
+ }
}
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/DummyTokenStream.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/DummyTokenStream.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/DummyTokenStream.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/DummyTokenStream.java Fri Jul 24 08:58:52 2009
@@ -24,45 +24,43 @@
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-public class DummyTokenStream extends TokenStream {
- private int count = 0;
-
- private String tokenValue;
-
- private int distance;
-
- private int number;
-
- private int begin;
-
- private int end;
-
- public DummyTokenStream(String tokenValue, int distance, int number, int offset) {
- this.tokenValue = tokenValue;
- this.distance = distance;
- this.number = number;
-
- if (offset > 0) {
- count = offset;
- begin = count * (tokenValue.length() + 1);
- end = count * (tokenValue.length() + 1) + tokenValue.length();
- } else {
- begin = 0;
- end = tokenValue.length();
- }
- }
-
- public Token next() throws IOException {
- if (number <= count / distance)
- return null;
-
- Token token = new Token(tokenValue, begin, end);
-
- count += distance;
- begin += distance * (tokenValue.length() + 1);
- end += distance * (tokenValue.length() + 1);
-
- System.out.println(token);
- return token;
- }
-}
+public class DummyTokenStream extends TokenStream{
+ private int count = 0;
+ private String tokenValue;
+ private int distance;
+ private int number;
+ private int begin;
+ private int end;
+
+
+ public DummyTokenStream(String tokenValue, int distance, int number, int offset) {
+ this.tokenValue = tokenValue;
+ this.distance = distance;
+ this.number = number;
+
+
+ if( offset > 0 ){
+ count = offset;
+ begin = count * (tokenValue.length() + 1);
+ end = count * (tokenValue.length() + 1) + tokenValue.length();
+ }
+ else{
+ begin = 0;
+ end = tokenValue.length();
+ }
+ }
+
+ public Token next() throws IOException {
+ if( number <= count / distance )
+ return null;
+
+ Token token = new Token(tokenValue, begin, end);
+
+ count += distance;
+ begin+= distance* (tokenValue.length() + 1);
+ end+= distance* (tokenValue.length() + 1);
+
+ System.out.println(token);
+ return token;
+ }
+}
\ No newline at end of file
Modified: incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/MockCollectionReader.java
URL: http://svn.apache.org/viewvc/incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/MockCollectionReader.java?rev=797373&r1=797372&r2=797373&view=diff
==============================================================================
--- incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/MockCollectionReader.java (original)
+++ incubator/uima/sandbox/trunk/Lucas/src/test/java/org/apache/uima/lucas/indexer/test/util/MockCollectionReader.java Fri Jul 24 08:58:52 2009
@@ -28,17 +28,24 @@
public class MockCollectionReader extends CollectionReader_ImplBase {
- public void getNext(CAS cas) throws IOException, CollectionException {
- }
+ public void getNext(CAS arg0) throws IOException, CollectionException {
+ // TODO Auto-generated method stub
- public void close() throws IOException {
- }
+ }
+
+ public void close() throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ public Progress[] getProgress() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public boolean hasNext() throws IOException, CollectionException {
+ // TODO Auto-generated method stub
+ return false;
+ }
- public Progress[] getProgress() {
- return null;
- }
-
- public boolean hasNext() throws IOException, CollectionException {
- return false;
- }
}