You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2017/01/02 13:53:20 UTC
opennlp git commit: Remove deprecated code from util package
Repository: opennlp
Updated Branches:
refs/heads/908 [created] 60a534cc5
Remove deprecated code from util package
See issue OPENNLP-906
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/60a534cc
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/60a534cc
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/60a534cc
Branch: refs/heads/908
Commit: 60a534cc53c5b3a4d3d7d18965745534cd394803
Parents: 3c39ef7
Author: J�rn Kottmann <jo...@apache.org>
Authored: Mon Jan 2 14:52:37 2017 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Jan 2 14:52:37 2017 +0100
----------------------------------------------------------------------
.../java/opennlp/tools/util/BeamSearch.java | 201 ----------------
.../java/opennlp/tools/util/CountedSet.java | 241 -------------------
.../main/java/opennlp/tools/util/TreeHeap.java | 95 --------
.../featuregen/AdaptiveFeatureGenerator.java | 9 +-
.../FastTokenClassFeatureGenerator.java | 119 ---------
.../featuregen/FeatureGeneratorAdapter.java | 37 ---
.../featuregen/FeatureGeneratorFactory.java | 51 ----
.../util/featuregen/FeatureGeneratorUtil.java | 58 ++++-
.../featuregen/PreviousMapFeatureGenerator.java | 5 +-
.../PreviousTwoMapFeatureGenerator.java | 5 +-
.../tools/util/model/ClassSerializer.java | 66 -----
.../FeatureGeneratorFactorySerializer.java | 56 -----
.../java/opennlp/tools/ml/BeamSearchTest.java | 213 ++++++++++++++++
.../java/opennlp/tools/util/BeamSearchTest.java | 204 ----------------
14 files changed, 277 insertions(+), 1083 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java b/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
deleted file mode 100644
index 95cbea9..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util;
-
-import java.util.Arrays;
-import java.util.List;
-
-import opennlp.tools.ml.model.MaxentModel;
-
-/**
- * Performs k-best search over sequence. This is based on the description in
- * Ratnaparkhi (1998), PhD diss, Univ. of Pennsylvania.
- *
- * @see Sequence
- * @see SequenceValidator
- * @see BeamSearchContextGenerator
- *
- *
- * @deprecated use opennlp.tools.ml.BeamSearch instead
- */
-@Deprecated
-public class BeamSearch<T> {
-
- private static final Object[] EMPTY_ADDITIONAL_CONTEXT = new Object[0];
-
- protected int size;
- protected BeamSearchContextGenerator<T> cg;
- protected MaxentModel model;
- private SequenceValidator<T> validator;
-
- private double[] probs;
- private Cache<String[], double[]> contextsCache;
- private static final int zeroLog = -100000;
-
- /**
- * Creates new search object.
- *
- * @param size The size of the beam (k).
- * @param cg the context generator for the model.
- * @param model the model for assigning probabilities to the sequence outcomes.
- */
- public BeamSearch(int size, BeamSearchContextGenerator<T> cg, MaxentModel model) {
- this(size, cg, model, null, 0);
- }
-
- public BeamSearch(int size, BeamSearchContextGenerator<T> cg, MaxentModel model,
- int cacheSize) {
- this (size, cg, model, null, cacheSize);
- }
-
- public BeamSearch(int size, BeamSearchContextGenerator<T> cg, MaxentModel model,
- SequenceValidator<T> validator, int cacheSize) {
-
- this.size = size;
- this.cg = cg;
- this.model = model;
- this.validator = validator;
-
- if (cacheSize > 0) {
- contextsCache = new Cache<>(cacheSize);
- }
-
- this.probs = new double[model.getNumOutcomes()];
- }
-
- /**
- * Note:
- * This method will be private in the future because clients can now
- * pass a validator to validate the sequence.
- *
- * @see SequenceValidator
- */
- private boolean validSequence(int i, T[] inputSequence, String[] outcomesSequence, String outcome) {
- return validator == null || validator.validSequence(i, inputSequence, outcomesSequence, outcome);
- }
-
- public Sequence[] bestSequences(int numSequences, T[] sequence, Object[] additionalContext) {
- return bestSequences(numSequences, sequence, additionalContext, zeroLog);
- }
-
- /**
- * Returns the best sequence of outcomes based on model for this object.
- *
- * @param numSequences The maximum number of sequences to be returned.
- * @param sequence The input sequence.
- * @param additionalContext An Object[] of additional context. This is passed to the context generator blindly with the assumption that the context are appropiate.
- * @param minSequenceScore A lower bound on the score of a returned sequence.
- * @return An array of the top ranked sequences of outcomes.
- */
- public Sequence[] bestSequences(int numSequences, T[] sequence, Object[] additionalContext, double minSequenceScore) {
-
- Heap<Sequence> prev = new ListHeap<>(size);
- Heap<Sequence> next = new ListHeap<>(size);
- Heap<Sequence> tmp;
- prev.add(new Sequence());
-
- if (additionalContext == null) {
- additionalContext = EMPTY_ADDITIONAL_CONTEXT;
- }
-
- for (int i = 0; i < sequence.length; i++) {
- int sz = Math.min(size, prev.size());
-
- for (int sc = 0; prev.size() > 0 && sc < sz; sc++) {
- Sequence top = prev.extract();
- List<String> tmpOutcomes = top.getOutcomes();
- String[] outcomes = tmpOutcomes.toArray(new String[tmpOutcomes.size()]);
- String[] contexts = cg.getContext(i, sequence, outcomes, additionalContext);
- double[] scores;
- if (contextsCache != null) {
- scores = contextsCache.get(contexts);
- if (scores == null) {
- scores = model.eval(contexts, probs);
- contextsCache.put(contexts,scores);
- }
- }
- else {
- scores = model.eval(contexts, probs);
- }
-
- double[] temp_scores = new double[scores.length];
- System.arraycopy(scores, 0, temp_scores, 0, scores.length);
-
- Arrays.sort(temp_scores);
-
- double min = temp_scores[Math.max(0,scores.length-size)];
-
- for (int p = 0; p < scores.length; p++) {
- if (scores[p] < min)
- continue; //only advance first "size" outcomes
- String out = model.getOutcome(p);
- if (validSequence(i, sequence, outcomes, out)) {
- Sequence ns = new Sequence(top, out, scores[p]);
- if (ns.getScore() > minSequenceScore) {
- next.add(ns);
- }
- }
- }
-
- if (next.size() == 0) {//if no advanced sequences, advance all valid
- for (int p = 0; p < scores.length; p++) {
- String out = model.getOutcome(p);
- if (validSequence(i, sequence, outcomes, out)) {
- Sequence ns = new Sequence(top, out, scores[p]);
- if (ns.getScore() > minSequenceScore) {
- next.add(ns);
- }
- }
- }
- }
- }
-
- // make prev = next; and re-init next (we reuse existing prev set once we clear it)
- prev.clear();
- tmp = prev;
- prev = next;
- next = tmp;
- }
-
- int numSeq = Math.min(numSequences, prev.size());
- Sequence[] topSequences = new Sequence[numSeq];
-
- for (int seqIndex = 0; seqIndex < numSeq; seqIndex++) {
- topSequences[seqIndex] = prev.extract();
- }
-
- return topSequences;
- }
-
- /**
- * Returns the best sequence of outcomes based on model for this object.
- *
- * @param sequence The input sequence.
- * @param additionalContext An Object[] of additional context. This is passed to the context generator blindly with the assumption that the context are appropiate.
- *
- * @return The top ranked sequence of outcomes or null if no sequence could be found
- */
- public Sequence bestSequence(T[] sequence, Object[] additionalContext) {
- Sequence sequences[] = bestSequences(1, sequence, additionalContext,zeroLog);
-
- if (sequences.length > 0)
- return sequences[0];
- else
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/CountedSet.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/CountedSet.java b/opennlp-tools/src/main/java/opennlp/tools/util/CountedSet.java
deleted file mode 100644
index 53659b1..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/CountedSet.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package opennlp.tools.util;
-
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Set which counts the number of times a values are added to it.
- * This value can be accessed with the #getCount method.
- *
- * @deprecated this class is no longer used in OpenNLP and will be removed
- */
-@Deprecated
-public class CountedSet<E> implements Set<E> {
-
- private Map<E, Integer> cset;
-
- /**
- * Creates a new counted set.
- */
- public CountedSet() {
- cset = new HashMap<E, Integer>();
- }
-
- /**
- * Creates a new counted set of the specified initial size.
- *
- * @param size The initial size of this set.
- */
- public CountedSet(int size) {
- cset = new HashMap<E, Integer>(size);
-
- }
-
- public boolean add(E o) {
- Integer count = cset.get(o);
- if ( count == null ) {
- cset.put(o, 1);
- return true;
- }
- else {
- cset.put(o, count + 1);
- return false;
- }
- }
-
- /**
- * Reduces the count associated with this object by 1. If this causes the count
- * to become 0, then the object is removed form the set.
- *
- * @param o The object whose count is being reduced.
- */
- public void subtract(E o) {
- Integer count = cset.get(o);
- if ( count != null ) {
- int c = count -1;
- if (c == 0) {
- cset.remove(o);
- }
- else {
- cset.put(o, c);
- }
- }
- }
-
- /**
- * Assigns the specified object the specified count in the set.
- *
- * @param o The object to be added or updated in the set.
- * @param c The count of the specified object.
- */
- public void setCount(E o, int c) {
- cset.put(o, c);
- }
-
- /**
- * Return the count of the specified object.
- *
- * @param o the object whose count needs to be determined.
- * @return the count of the specified object.
- */
- public int getCount(E o) {
- Integer count = cset.get(o);
- if ( count == null ) {
- return 0;
- }
- else {
- return count;
- }
- }
-
- /**
- * This methods is deprecated use opennlp.toolsdictionary.serialization
- * package for writing a {@link CountedSet}.
- *
- * @param fileName
- * @param countCutoff
- */
- @Deprecated
- public void write(String fileName,int countCutoff) {
- write(fileName,countCutoff," ");
- }
-
- /**
- * This methods is deprecated use opennlp.toolsdictionary.serialization
- * package for writing a {@link CountedSet}.
- *
- * @param fileName
- * @param countCutoff
- * @param delim
- */
- @Deprecated
- public void write(String fileName,int countCutoff,String delim) {
- write(fileName,countCutoff,delim,null);
- }
-
- /**
- * This methods is deprecated use opennlp.toolsdictionary.serialization
- * package for writing a {@link CountedSet}.
- *
- * @param fileName
- * @param countCutoff
- * @param delim
- * @param encoding
- */
- @Deprecated
- public void write(String fileName,int countCutoff,String delim,String encoding) {
- PrintWriter out = null;
- try{
- if (encoding != null) {
- out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(fileName),encoding));
- }
- else {
- out = new PrintWriter(new FileWriter(fileName));
- }
-
- for (Iterator<E> e = cset.keySet().iterator(); e.hasNext();) {
- E key = e.next();
- int count = this.getCount(key);
- if ( count >= countCutoff ) {
- out.println(count + delim + key);
- }
- }
- out.close();
- }
- catch (IOException e) {
- System.err.println(e);
- }
- }
-
- public boolean addAll(Collection<? extends E> c) {
- boolean changed = false;
-
- for (E element : c) {
- changed = changed || add(element);
- }
-
- return changed;
- }
-
- public void clear() {
- cset.clear();
- }
-
- public boolean contains(Object o) {
- return cset.keySet().contains(o);
- }
-
- public boolean containsAll(Collection<?> c) {
- return cset.keySet().containsAll(c);
- }
-
- public boolean isEmpty() {
- return cset.isEmpty();
- }
-
- public Iterator<E> iterator() {
- return cset.keySet().iterator();
- }
-
- public boolean remove(Object o) {
- return cset.remove(o) != null;
- }
-
- public boolean removeAll(Collection<?> c) {
- boolean changed =false;
- for (Iterator<E> ki = cset.keySet().iterator(); ki.hasNext();) {
- changed = changed || cset.remove(ki.next()) != null;
- }
- return changed;
- }
-
- public boolean retainAll(Collection<?> c) {
- boolean changed = false;
- for (Iterator<E> ki = cset.keySet().iterator();ki.hasNext();) {
- Object key = ki.next();
- if (!c.contains(key)) {
- cset.remove(key);
- changed = true;
- }
- }
- return changed;
- }
-
- public int size() {
- return cset.size();
- }
-
- public Object[] toArray() {
- return cset.keySet().toArray();
- }
-
- public <T> T[] toArray(T[] a) {
- return cset.keySet().toArray(a);
- }
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/TreeHeap.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/TreeHeap.java b/opennlp-tools/src/main/java/opennlp/tools/util/TreeHeap.java
deleted file mode 100644
index f410ffd..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/TreeHeap.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util;
-
-import java.util.Iterator;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-/**
- * An implementation of the Heap interface based on {@link java.util.SortedSet}.
- * This implementation will not allow multiple objects which are equal to be added to the heap.
- * Only use this implementation when object in the heap can be totally ordered (no duplicates).
- *
- * @deprecated not used anymore, when there is need for a heap use ListHeap instead
- */
-@Deprecated
-public class TreeHeap<E> implements Heap<E> {
-
- private SortedSet<E> tree;
-
- /**
- * Creates a new tree heap.
- */
- public TreeHeap() {
- tree = new TreeSet<E>();
- }
-
- /**
- * Creates a new tree heap of the specified size.
- * @param size The size of the new tree heap.
- */
- public TreeHeap(int size) {
- tree = new TreeSet<E>();
- }
-
- public E extract() {
- E rv = tree.first();
- tree.remove(rv);
- return rv;
- }
-
- public E first() {
- return tree.first();
- }
-
- public E last() {
- return tree.last();
- }
-
- public Iterator<E> iterator() {
- return tree.iterator();
- }
-
- public void add(E o) {
- tree.add(o);
- }
-
- public int size() {
- return tree.size();
- }
-
- public void clear() {
- tree.clear();
- }
-
- public boolean isEmpty(){
- return this.tree.isEmpty();
- }
-
- public static void main(String[] args) {
- Heap<Integer> heap = new TreeHeap<Integer>(5);
- for (int ai=0;ai<args.length;ai++){
- heap.add(Integer.parseInt(args[ai]));
- }
- while (!heap.isEmpty()) {
- System.out.print(heap.extract()+" ");
- }
- System.out.println();
- }
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
index ce8e9e4..85d9df9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
@@ -24,19 +24,12 @@ import java.util.List;
* An interface for generating features for name entity identification and for
* updating document level contexts.
* <p>
- * Most implementors do not need the adaptive functionality of this
- * interface, they should extend the {@link FeatureGeneratorAdapter} class instead.
- * <p>
+ *
* <b>Note:</b><br>
* Feature generation is not thread safe and a instance of a feature generator
* must only be called from one thread. The resources used by a feature
* generator are typically shared between man instances of features generators
* which are called from many threads and have to be thread safe.
- * If that is not possible the {@link FeatureGeneratorFactory} must make a copy
- * of the resource object for each feature generator instance.
- *
- * @see FeatureGeneratorAdapter
- * @see FeatureGeneratorFactory
*/
public interface AdaptiveFeatureGenerator {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
deleted file mode 100644
index 5c82733..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util.featuregen;
-
-import java.util.List;
-import java.util.regex.Pattern;
-
-import opennlp.tools.util.StringUtil;
-
-
-
-/**
- * Generates features for different for the class of the token.
- *
- * @deprecated Use {@link TokenClassFeatureGenerator} instead!
- */
-@Deprecated
-public class FastTokenClassFeatureGenerator implements AdaptiveFeatureGenerator {
-
- private static final String TOKEN_CLASS_PREFIX = "wc";
- private static final String TOKEN_AND_CLASS_PREFIX = "w&c";
-
- private static Pattern capPeriod;
-
- static {
- capPeriod = Pattern.compile("^[A-Z]\\.$");
- }
-
- private boolean generateWordAndClassFeature;
-
-
-
- public FastTokenClassFeatureGenerator() {
- this(false);
- }
-
- public FastTokenClassFeatureGenerator(boolean genearteWordAndClassFeature) {
- this.generateWordAndClassFeature = genearteWordAndClassFeature;
- }
-
-
- public static String tokenFeature(String token) {
-
- StringPattern pattern = StringPattern.recognize(token);
-
- String feat;
- if (pattern.isAllLowerCaseLetter()) {
- feat = "lc";
- }
- else if (pattern.digits() == 2) {
- feat = "2d";
- }
- else if (pattern.digits() == 4) {
- feat = "4d";
- }
- else if (pattern.containsDigit()) {
- if (pattern.containsLetters()) {
- feat = "an";
- }
- else if (pattern.containsHyphen()) {
- feat = "dd";
- }
- else if (pattern.containsSlash()) {
- feat = "ds";
- }
- else if (pattern.containsComma()) {
- feat = "dc";
- }
- else if (pattern.containsPeriod()) {
- feat = "dp";
- }
- else {
- feat = "num";
- }
- }
- else if (pattern.isAllCapitalLetter() && token.length() == 1) {
- feat = "sc";
- }
- else if (pattern.isAllCapitalLetter()) {
- feat = "ac";
- }
- else if (capPeriod.matcher(token).find()) {
- feat = "cp";
- }
- else if (pattern.isInitialCapitalLetter()) {
- feat = "ic";
- }
- else {
- feat = "other";
- }
-
- return (feat);
- }
-
-
- public void createFeatures(List<String> features, String[] tokens, int index, String[] preds) {
- String wordClass = tokenFeature(tokens[index]);
- features.add(TOKEN_CLASS_PREFIX + "=" + wordClass);
-
- if (generateWordAndClassFeature) {
- features.add(TOKEN_AND_CLASS_PREFIX + "=" + StringUtil.toLowerCase(tokens[index]) +","+wordClass);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
deleted file mode 100644
index 8f444ce..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util.featuregen;
-
-
-/**
- * This class provides empty implementations of some of the optional methods in
- * {@link AdditionalContextFeatureGenerator} to make implementing feature generators
- * easier.
- *
- * @deprecated use AdaptiveFeatureGenerator instead
- */
-@Deprecated // in 1.7.0
-public abstract class FeatureGeneratorAdapter implements AdaptiveFeatureGenerator {
-
- public void updateAdaptiveData(String[] tokens, String[] outcomes) {
- }
-
- public void clearAdaptiveData() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorFactory.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorFactory.java
deleted file mode 100644
index e62abad..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package opennlp.tools.util.featuregen;
-
-/**
- * The {@link FeatureGeneratorFactory} interface is factory for {@link AdaptiveFeatureGenerator}s.
- * <p>
- * <b>Note:</b><br>
- * All implementing classes must be thread safe.
- *
- * @see AdaptiveFeatureGenerator
- * @see FeatureGeneratorResourceProvider
- *
- *
- * @deprecated do not use this interface, will be removed!
- */
-@Deprecated
-public interface FeatureGeneratorFactory {
-
- /**
- * Constructs a new {@link AdaptiveFeatureGenerator}.
- * <p>
- * <b>Note:</b><br>
- * It is assumed that all resource objects are thread safe and can be shared
- * between multiple instances of feature generators. If that is not the
- * case the implementor should make a copy of the resource object.
- * All resource objects that are included in OpenNLP can be assumed to be thread safe.
- *
- * @param resourceProvider provides access to resources which are needed for feature generation.
- *
- * @return the newly created feature generator
- */
- @Deprecated
- AdaptiveFeatureGenerator createFeatureGenerator(FeatureGeneratorResourceProvider resourceProvider);
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
index 8a1e470..dfcf10d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
@@ -18,11 +18,18 @@
package opennlp.tools.util.featuregen;
+import java.util.regex.Pattern;
+
/**
* This class provide common utilities for feature generation.
*/
public class FeatureGeneratorUtil {
+ private static final String TOKEN_CLASS_PREFIX = "wc";
+ private static final String TOKEN_AND_CLASS_PREFIX = "w&c";
+
+ private static final Pattern capPeriod = Pattern.compile("^[A-Z]\\.$");
+
/**
* Generates a class name for the specified token.
* The classes are as follows where the first matching class is used:
@@ -45,6 +52,55 @@ public class FeatureGeneratorUtil {
* @return The class name that the specified token belongs in.
*/
public static String tokenFeature(String token) {
- return FastTokenClassFeatureGenerator.tokenFeature(token);
+
+ StringPattern pattern = StringPattern.recognize(token);
+
+ String feat;
+ if (pattern.isAllLowerCaseLetter()) {
+ feat = "lc";
+ }
+ else if (pattern.digits() == 2) {
+ feat = "2d";
+ }
+ else if (pattern.digits() == 4) {
+ feat = "4d";
+ }
+ else if (pattern.containsDigit()) {
+ if (pattern.containsLetters()) {
+ feat = "an";
+ }
+ else if (pattern.containsHyphen()) {
+ feat = "dd";
+ }
+ else if (pattern.containsSlash()) {
+ feat = "ds";
+ }
+ else if (pattern.containsComma()) {
+ feat = "dc";
+ }
+ else if (pattern.containsPeriod()) {
+ feat = "dp";
+ }
+ else {
+ feat = "num";
+ }
+ }
+ else if (pattern.isAllCapitalLetter() && token.length() == 1) {
+ feat = "sc";
+ }
+ else if (pattern.isAllCapitalLetter()) {
+ feat = "ac";
+ }
+ else if (capPeriod.matcher(token).find()) {
+ feat = "cp";
+ }
+ else if (pattern.isInitialCapitalLetter()) {
+ feat = "ic";
+ }
+ else {
+ feat = "other";
+ }
+
+ return (feat);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousMapFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousMapFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousMapFeatureGenerator.java
index 0426897..53efe8e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousMapFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousMapFeatureGenerator.java
@@ -22,11 +22,12 @@ import java.util.List;
import java.util.Map;
/**
- * This {@link FeatureGeneratorAdapter} generates features indicating the outcome associated with a previously occuring word.
+ * This {@link AdaptiveFeatureGenerator} generates features indicating the
+ * outcome associated with a previously occuring word.
*/
public class PreviousMapFeatureGenerator implements AdaptiveFeatureGenerator {
- private Map<String, String> previousMap = new HashMap<String, String>();
+ private Map<String, String> previousMap = new HashMap<>();
public void createFeatures(List<String> features, String[] tokens, int index, String[] preds) {
features.add("pd=" + previousMap.get(tokens[index]));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousTwoMapFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousTwoMapFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousTwoMapFeatureGenerator.java
index 842e2ac..bcb6aec 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousTwoMapFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PreviousTwoMapFeatureGenerator.java
@@ -22,11 +22,12 @@ import java.util.List;
import java.util.Map;
/**
- * This {@link FeatureGeneratorAdapter} generates features indicating the outcome associated with two previously occuring words.
+ * This {@link AdaptiveFeatureGenerator} generates features indicating the
+ * outcome associated with two previously occuring words.
*/
public class PreviousTwoMapFeatureGenerator implements AdaptiveFeatureGenerator {
- private Map<String, String> previousMap = new HashMap<String, String>();
+ private Map<String, String> previousMap = new HashMap<>();
/**
* Generates previous decision features for the token based on contents of the previous map.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java
deleted file mode 100644
index fe8a6f8..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package opennlp.tools.util.model;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import opennlp.tools.util.InvalidFormatException;
-
-@Deprecated
-public class ClassSerializer implements ArtifactSerializer<Class<?>> {
-
- private static final String CLASS_SEARCH_NAME = "ClassSearchName";
-
- private byte[] classBytes;
-
- private static Class<?> loadClass(final byte[] classBytes)
- throws InvalidFormatException {
-
- ClassLoader loader = new ClassLoader() {
- @Override
- protected Class<?> findClass(String name) throws ClassNotFoundException {
- if (CLASS_SEARCH_NAME.equals(name))
- return defineClass(null, classBytes, 0, classBytes.length);
- else
- return super.findClass(name);
- }
- };
-
- try {
- return loader.loadClass(CLASS_SEARCH_NAME);
- } catch (ClassNotFoundException e) {
- throw new InvalidFormatException(e);
- }
- }
-
- public Class<?> create(InputStream in) throws IOException,
- InvalidFormatException {
- classBytes = ModelUtil.read(in);
-
- Class<?> factoryClass = loadClass(classBytes);
-
- return factoryClass;
- }
-
- public void serialize(Class<?> artifact, OutputStream out) throws IOException {
- out.write(classBytes);
- }
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java
deleted file mode 100644
index b6565f5..0000000
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package opennlp.tools.util.model;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import opennlp.tools.util.InvalidFormatException;
-import opennlp.tools.util.featuregen.FeatureGeneratorFactory;
-
-@Deprecated
-public class FeatureGeneratorFactorySerializer
- implements ArtifactSerializer<FeatureGeneratorFactory>{
-
- private ClassSerializer classSerializer;
-
- public FeatureGeneratorFactorySerializer() {
- classSerializer = new ClassSerializer();
- }
-
- public FeatureGeneratorFactory create(InputStream in) throws IOException,
- InvalidFormatException {
-
- Class<?> generatorFactoryClass = classSerializer.create(in);
-
- try {
- return (FeatureGeneratorFactory) generatorFactoryClass.newInstance();
- } catch (InstantiationException e) {
- throw new InvalidFormatException(e);
- } catch (IllegalAccessException e) {
- throw new InvalidFormatException(e);
- }
- }
-
- public void serialize(FeatureGeneratorFactory artifact, OutputStream out)
- throws IOException {
- classSerializer.serialize(artifact.getClass(), out);
- }
-}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/test/java/opennlp/tools/ml/BeamSearchTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/ml/BeamSearchTest.java b/opennlp-tools/src/test/java/opennlp/tools/ml/BeamSearchTest.java
new file mode 100644
index 0000000..3ac8467
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/ml/BeamSearchTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.ml;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import opennlp.tools.ml.model.MaxentModel;
+
+import opennlp.tools.util.BeamSearchContextGenerator;
+import opennlp.tools.util.Sequence;
+import org.junit.Test;
+
+public class BeamSearchTest {
+
+ static class IdentityFeatureGenerator implements BeamSearchContextGenerator<String> {
+
+ private String[] outcomeSequence;
+
+ IdentityFeatureGenerator(String outcomeSequence[]) {
+ this.outcomeSequence = outcomeSequence;
+ }
+
+ public String[] getContext(int index, String[] sequence,
+ String[] priorDecisions, Object[] additionalContext) {
+ return new String[] {outcomeSequence[index]};
+ }
+ }
+
+
+ static class IdentityModel implements MaxentModel {
+
+ private String[] outcomes;
+
+ private Map<String, Integer> outcomeIndexMap = new HashMap<String, Integer>();
+
+ private double bestOutcomeProb = 0.8d;
+ private double otherOutcomeProb;
+
+ IdentityModel(String outcomes[]) {
+ this.outcomes = outcomes;
+
+ for (int i = 0; i < outcomes.length; i++) {
+ outcomeIndexMap.put(outcomes[i], i);
+ }
+
+ otherOutcomeProb = 0.2d / (outcomes.length - 1);
+ }
+
+ public double[] eval(String[] context) {
+
+ double probs[] = new double[outcomes.length];
+
+ for (int i = 0; i < probs.length; i++) {
+ if (outcomes[i].equals(context[0])) {
+ probs[i] = bestOutcomeProb;
+ }
+ else {
+ probs[i] = otherOutcomeProb;
+ }
+ }
+
+ return probs;
+ }
+
+ public double[] eval(String[] context, double[] probs) {
+ return eval(context);
+ }
+
+ public double[] eval(String[] context, float[] values) {
+ return eval(context);
+ }
+
+ public String getAllOutcomes(double[] outcomes) {
+ return null;
+ }
+
+ public String getBestOutcome(double[] outcomes) {
+ return null;
+ }
+
+ public Object[] getDataStructures() {
+ return null;
+ }
+
+ public int getIndex(String outcome) {
+ return 0;
+ }
+
+ public int getNumOutcomes() {
+ return outcomes.length;
+ }
+
+ public String getOutcome(int i) {
+ return outcomes[i];
+ }
+ }
+
+ /**
+ * Tests that beam search does not fail to detect an empty sequence.
+ */
+ @Test
+ public void testBestSequenceZeroLengthInput() {
+
+ String sequence[] = new String[0];
+ BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
+
+ String outcomes[] = new String[] {"1", "2", "3"};
+ MaxentModel model = new IdentityModel(outcomes);
+
+ BeamSearch<String> bs = new BeamSearch<>(3, model);
+
+ Sequence seq = bs.bestSequence(sequence, null, cg,
+ (int i, String[] inputSequence, String[] outcomesSequence,
+ String outcome) -> {return true;});
+ assertNotNull(seq);
+ assertEquals(sequence.length, seq.getOutcomes().size());
+ }
+
+ /**
+ * Tests finding a sequence of length one.
+ */
+ @Test
+ public void testBestSequenceOneElementInput() {
+ String sequence[] = {"1"};
+ BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
+
+ String outcomes[] = new String[] {"1", "2", "3"};
+ MaxentModel model = new IdentityModel(outcomes);
+
+ BeamSearch<String> bs = new BeamSearch<>(3, model);
+
+ Sequence seq = bs.bestSequence(sequence, null, cg,
+ (int i, String[] inputSequence, String[] outcomesSequence,
+ String outcome) -> {return true;});
+
+ assertNotNull(seq);
+ assertEquals(sequence.length, seq.getOutcomes().size());
+ assertEquals("1", seq.getOutcomes().get(0));
+ }
+
+ /**
+ * Tests finding the best sequence on a short input sequence.
+ */
+ @Test
+ public void testBestSequence() {
+ String sequence[] = {"1", "2", "3", "2", "1"};
+ BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
+
+ String outcomes[] = new String[] {"1", "2", "3"};
+ MaxentModel model = new IdentityModel(outcomes);
+
+ BeamSearch<String> bs = new BeamSearch<String>(2, model);
+
+ Sequence seq = bs.bestSequence(sequence, null, cg,
+ (int i, String[] inputSequence, String[] outcomesSequence,
+ String outcome) -> {return true;});
+
+ assertNotNull(seq);
+ assertEquals(sequence.length, seq.getOutcomes().size());
+ assertEquals("1", seq.getOutcomes().get(0));
+ assertEquals("2", seq.getOutcomes().get(1));
+ assertEquals("3", seq.getOutcomes().get(2));
+ assertEquals("2", seq.getOutcomes().get(3));
+ assertEquals("1", seq.getOutcomes().get(4));
+ }
+
+ /**
+ * Tests finding the best sequence on a short input sequence.
+ */
+ @Test
+ public void testBestSequenceWithValidator() {
+ String sequence[] = {"1", "2", "3", "2", "1"};
+ BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
+
+ String outcomes[] = new String[] {"1", "2", "3"};
+ MaxentModel model = new IdentityModel(outcomes);
+
+ BeamSearch<String> bs = new BeamSearch<>(2, model, 0);
+
+ Sequence seq = bs.bestSequence(sequence, null, cg,
+ (int i, String[] inputSequence,
+ String[] outcomesSequence, String outcome) -> {
+ return !"2".equals(outcome);
+ });
+ assertNotNull(seq);
+ assertEquals(sequence.length, seq.getOutcomes().size());
+ assertEquals("1", seq.getOutcomes().get(0));
+ assertNotSame("2", seq.getOutcomes().get(1));
+ assertEquals("3", seq.getOutcomes().get(2));
+ assertNotSame("2", seq.getOutcomes().get(3));
+ assertEquals("1", seq.getOutcomes().get(4));
+ }
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/60a534cc/opennlp-tools/src/test/java/opennlp/tools/util/BeamSearchTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/BeamSearchTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/BeamSearchTest.java
deleted file mode 100644
index 83f6552..0000000
--- a/opennlp-tools/src/test/java/opennlp/tools/util/BeamSearchTest.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import opennlp.tools.ml.model.MaxentModel;
-
-import org.junit.Test;
-
-public class BeamSearchTest {
-
- static class IdentityFeatureGenerator implements BeamSearchContextGenerator<String> {
-
- private String[] outcomeSequence;
-
- IdentityFeatureGenerator(String outcomeSequence[]) {
- this.outcomeSequence = outcomeSequence;
- }
-
- public String[] getContext(int index, String[] sequence,
- String[] priorDecisions, Object[] additionalContext) {
- return new String[] {outcomeSequence[index]};
- }
- }
-
-
- static class IdentityModel implements MaxentModel {
-
- private String[] outcomes;
-
- private Map<String, Integer> outcomeIndexMap = new HashMap<String, Integer>();
-
- private double bestOutcomeProb = 0.8d;
- private double otherOutcomeProb;
-
- IdentityModel(String outcomes[]) {
- this.outcomes = outcomes;
-
- for (int i = 0; i < outcomes.length; i++) {
- outcomeIndexMap.put(outcomes[i], i);
- }
-
- otherOutcomeProb = 0.2d / (outcomes.length - 1);
- }
-
- public double[] eval(String[] context) {
-
- double probs[] = new double[outcomes.length];
-
- for (int i = 0; i < probs.length; i++) {
- if (outcomes[i].equals(context[0])) {
- probs[i] = bestOutcomeProb;
- }
- else {
- probs[i] = otherOutcomeProb;
- }
- }
-
- return probs;
- }
-
- public double[] eval(String[] context, double[] probs) {
- return eval(context);
- }
-
- public double[] eval(String[] context, float[] values) {
- return eval(context);
- }
-
- public String getAllOutcomes(double[] outcomes) {
- return null;
- }
-
- public String getBestOutcome(double[] outcomes) {
- return null;
- }
-
- public Object[] getDataStructures() {
- return null;
- }
-
- public int getIndex(String outcome) {
- return 0;
- }
-
- public int getNumOutcomes() {
- return outcomes.length;
- }
-
- public String getOutcome(int i) {
- return outcomes[i];
- }
- }
-
- /**
- * Tests that beam search does not fail to detect an empty sequence.
- */
- @Test
- public void testBestSequenceZeroLengthInput() {
-
- String sequence[] = new String[0];
- BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
-
- String outcomes[] = new String[] {"1", "2", "3"};
- MaxentModel model = new IdentityModel(outcomes);
-
- BeamSearch<String> bs = new BeamSearch<String>(3, cg, model);
-
- Sequence seq = bs.bestSequence(sequence, null);
- assertNotNull(seq);
- assertEquals(sequence.length, seq.getOutcomes().size());
- }
-
- /**
- * Tests finding a sequence of length one.
- */
- @Test
- public void testBestSequenceOneElementInput() {
- String sequence[] = {"1"};
- BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
-
- String outcomes[] = new String[] {"1", "2", "3"};
- MaxentModel model = new IdentityModel(outcomes);
-
- BeamSearch<String> bs = new BeamSearch<String>(3, cg, model);
-
- Sequence seq = bs.bestSequence(sequence, null);
- assertNotNull(seq);
- assertEquals(sequence.length, seq.getOutcomes().size());
- assertEquals("1", seq.getOutcomes().get(0));
- }
-
- /**
- * Tests finding the best sequence on a short input sequence.
- */
- @Test
- public void testBestSequence() {
- String sequence[] = {"1", "2", "3", "2", "1"};
- BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
-
- String outcomes[] = new String[] {"1", "2", "3"};
- MaxentModel model = new IdentityModel(outcomes);
-
- BeamSearch<String> bs = new BeamSearch<String>(2, cg, model);
-
- Sequence seq = bs.bestSequence(sequence, null);
- assertNotNull(seq);
- assertEquals(sequence.length, seq.getOutcomes().size());
- assertEquals("1", seq.getOutcomes().get(0));
- assertEquals("2", seq.getOutcomes().get(1));
- assertEquals("3", seq.getOutcomes().get(2));
- assertEquals("2", seq.getOutcomes().get(3));
- assertEquals("1", seq.getOutcomes().get(4));
- }
-
- /**
- * Tests finding the best sequence on a short input sequence.
- */
- @Test
- public void testBestSequenceWithValidator() {
- String sequence[] = {"1", "2", "3", "2", "1"};
- BeamSearchContextGenerator<String> cg = new IdentityFeatureGenerator(sequence);
-
- String outcomes[] = new String[] {"1", "2", "3"};
- MaxentModel model = new IdentityModel(outcomes);
-
- BeamSearch<String> bs = new BeamSearch<String>(2, cg, model, new SequenceValidator<String>(){
-
- public boolean validSequence(int i, String[] inputSequence,
- String[] outcomesSequence, String outcome) {
- return !"2".equals(outcome);
- }}, 0);
-
- Sequence seq = bs.bestSequence(sequence, null);
- assertNotNull(seq);
- assertEquals(sequence.length, seq.getOutcomes().size());
- assertEquals("1", seq.getOutcomes().get(0));
- assertNotSame("2", seq.getOutcomes().get(1));
- assertEquals("3", seq.getOutcomes().get(2));
- assertNotSame("2", seq.getOutcomes().get(3));
- assertEquals("1", seq.getOutcomes().get(4));
- }
-}