You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/05/02 14:34:33 UTC
svn commit: r1591889 [10/14] - in /opennlp/trunk: opennlp-tools/lang/ml/
opennlp-tools/src/main/java/opennlp/tools/chunker/
opennlp-tools/src/main/java/opennlp/tools/cmdline/
opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/
opennlp-tools/src/...
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java Fri May 2 12:34:23 2014
@@ -72,7 +72,7 @@ import org.xml.sax.SAXException;
* {@link AdaptiveFeatureGenerator}. Elements can contain other
* elements in this case it is the responsibility of the mapped factory to process
* the child elements correctly. In some factories this leads to recursive
- * calls the
+ * calls the
* {@link GeneratorFactory.XmlFeatureGeneratorFactory#create(Element, FeatureGeneratorResourceProvider)}
* method.
*
@@ -242,17 +242,17 @@ public class GeneratorFactory {
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) throws InvalidFormatException {
-
+
String dictResourceKey = generatorElement.getAttribute("dict");
-
+
Object dictResource = resourceManager.getResource(dictResourceKey);
-
+
if (!(dictResource instanceof Dictionary)) {
throw new InvalidFormatException("No dictionary resource for key: " + dictResourceKey);
}
String prefix = generatorElement.getAttribute("prefix");
-
+
return new DictionaryFeatureGenerator(prefix, (Dictionary) dictResource);
}
@@ -272,7 +272,7 @@ public class GeneratorFactory {
factoryMap.put("docbegin", new DocumentBeginFeatureGenerator());
}
}
-
+
/**
* @see DictionaryFeatureGenerator
*/
@@ -280,16 +280,16 @@ public class GeneratorFactory {
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) throws InvalidFormatException {
-
+
String dictResourceKey = generatorElement.getAttribute("dict");
-
+
Object dictResource = resourceManager.getResource(dictResourceKey);
-
+
if (!(dictResource instanceof W2VClassesDictionary)) {
throw new InvalidFormatException("Not a W2VClassesDictionary resource for key: " + dictResourceKey);
}
-
+
return new WordClusterFeatureGenerator((W2VClassesDictionary) dictResource);
}
@@ -297,7 +297,7 @@ public class GeneratorFactory {
factoryMap.put("w2vwordcluster", new W2VClassesFeatureGeneratorFactory());
}
}
-
+
/**
* @see PreviousMapFeatureGenerator
*/
@@ -313,8 +313,8 @@ public class GeneratorFactory {
}
}
- // TODO: Add parameters ...
-
+ // TODO: Add parameters ...
+
/**
* @see SentenceFeatureGenerator
*/
@@ -322,18 +322,18 @@ public class GeneratorFactory {
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) {
-
+
String beginFeatureString = generatorElement.getAttribute("begin");
-
+
boolean beginFeature = true;
if (beginFeatureString.length() != 0)
beginFeature = Boolean.parseBoolean(beginFeatureString);
-
+
String endFeatureString = generatorElement.getAttribute("end");
boolean endFeature = true;
if (endFeatureString.length() != 0)
endFeature = Boolean.parseBoolean(endFeatureString);
-
+
return new SentenceFeatureGenerator(beginFeature, endFeature);
}
@@ -362,28 +362,28 @@ public class GeneratorFactory {
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) {
-
+
return new TokenFeatureGenerator();
}
-
+
static void register(Map<String, XmlFeatureGeneratorFactory> factoryMap) {
factoryMap.put("token", new TokenFeatureGeneratorFactory());
}
}
-
+
static class BigramNameFeatureGeneratorFactory implements XmlFeatureGeneratorFactory {
-
+
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) {
-
+
return new BigramNameFeatureGenerator();
}
-
+
static void register(Map<String, XmlFeatureGeneratorFactory> factoryMap) {
factoryMap.put("bigram", new BigramNameFeatureGeneratorFactory());
}
}
-
+
/**
* @see TokenPatternFeatureGenerator
*/
@@ -424,9 +424,9 @@ public class GeneratorFactory {
throw new InvalidFormatException("window feature generator must contain" +
" an aggregator element");
}
-
+
AdaptiveFeatureGenerator nestedGenerator = GeneratorFactory.createGenerator(nestedGeneratorElement, resourceManager);
-
+
String prevLengthString = generatorElement.getAttribute("prevLength");
int prevLength;
@@ -436,7 +436,7 @@ public class GeneratorFactory {
} catch (NumberFormatException e) {
throw new InvalidFormatException("prevLength attribute '" + prevLengthString + "' is not a number!", e);
}
-
+
String nextLengthString = generatorElement.getAttribute("nextLength");
int nextLength;
@@ -445,8 +445,8 @@ public class GeneratorFactory {
nextLength = Integer.parseInt(nextLengthString);
} catch (NumberFormatException e) {
throw new InvalidFormatException("nextLength attribute '" + nextLengthString + "' is not a number!", e);
- }
-
+ }
+
return new WindowFeatureGenerator(nestedGenerator, prevLength, nextLength);
}
@@ -469,61 +469,61 @@ public class GeneratorFactory {
factoryMap.put("prefix", new PrefixFeatureGeneratorFactory());
}
}
-
+
/**
* @see TokenPatternFeatureGenerator
*/
static class SuffixFeatureGeneratorFactory implements XmlFeatureGeneratorFactory {
-
+
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) {
return new SuffixFeatureGenerator();
}
-
+
static void register(Map<String, XmlFeatureGeneratorFactory> factoryMap) {
factoryMap.put("suffix", new SuffixFeatureGeneratorFactory());
}
}
-
+
// TODO: We have to support custom resources here. How does it work ?!
// Attributes get into a Map<String, String> properties
-
+
// How can serialization be supported ?!
// The model is loaded, and the manifest should contain all serializer classes registered for the
// resources by name.
// When training, the descriptor could be consulted first to register the serializers, and afterwards
// they are stored in the model.
-
+
static class CustomFeatureGeneratorFactory implements XmlFeatureGeneratorFactory {
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) throws InvalidFormatException {
-
+
String featureGeneratorClassName = generatorElement.getAttribute("class");
-
+
AdaptiveFeatureGenerator generator = ExtensionLoader.instantiateExtension(AdaptiveFeatureGenerator.class,
featureGeneratorClassName);
-
+
if (generator instanceof CustomFeatureGenerator) {
-
+
CustomFeatureGenerator customGenerator = (CustomFeatureGenerator) generator;
-
+
Map<String, String> properties = new HashMap<>();
NamedNodeMap attributes = generatorElement.getAttributes();
-
+
for (int i = 0; i < attributes.getLength(); i++) {
Node attribute = attributes.item(i);
if (!"class".equals(attribute.getNodeName())) {
properties.put(attribute.getNodeName(), attribute.getNodeValue());
}
}
-
+
if (resourceManager != null) {
customGenerator.init(properties, resourceManager);
}
}
-
+
return generator;
}
@@ -531,7 +531,7 @@ public class GeneratorFactory {
factoryMap.put("custom", new CustomFeatureGeneratorFactory());
}
}
-
+
private static Map<String, XmlFeatureGeneratorFactory> factories =
new HashMap<String, XmlFeatureGeneratorFactory>();
@@ -570,13 +570,13 @@ public class GeneratorFactory {
FeatureGeneratorResourceProvider resourceManager) throws InvalidFormatException {
String elementName = generatorElement.getTagName();
-
+
XmlFeatureGeneratorFactory generatorFactory = factories.get(elementName);
if (generatorFactory == null) {
throw new InvalidFormatException("Unexpected element: " + elementName);
}
-
+
return generatorFactory.create(generatorElement, resourceManager);
}
@@ -599,10 +599,10 @@ public class GeneratorFactory {
} catch (SAXException e) {
throw new InvalidFormatException("Descriptor is not valid XML!", e);
}
-
+
return xmlDescriptorDOM;
}
-
+
/**
* Creates an {@link AdaptiveFeatureGenerator} from an provided XML descriptor.
*
@@ -630,40 +630,40 @@ public class GeneratorFactory {
return createGenerator(generatorElement, resourceManager);
}
-
+
public static Map<String, ArtifactSerializer<?>> extractCustomArtifactSerializerMappings(
InputStream xmlDescriptorIn)
throws IOException, InvalidFormatException {
-
+
Map<String, ArtifactSerializer<?>> mapping = new HashMap<>();
-
+
org.w3c.dom.Document xmlDescriptorDOM = createDOM(xmlDescriptorIn);
-
+
XPath xPath = XPathFactory.newInstance().newXPath();
-
+
NodeList customElements;
try {
customElements = (NodeList) xPath.evaluate("custom", xmlDescriptorDOM.getDocumentElement(), XPathConstants.NODESET);
} catch (XPathExpressionException e) {
throw new IllegalStateException("The hard coded XPath expression should always be valid!");
}
-
+
for (int i = 0; i < customElements.getLength(); i++) {
-
+
if (customElements.item(i) instanceof Element) {
Element customElement = (Element) customElements.item(i);
-
+
// Note: The resource provider is not available at that point, to provide
// resources they need to be loaded first!
AdaptiveFeatureGenerator generator = createGenerator(customElement, null);
-
+
if (generator instanceof ArtifactToSerializerMapper) {
ArtifactToSerializerMapper mapper = (ArtifactToSerializerMapper) generator;
mapping.putAll(mapper.getArtifactSerializerMapping());
}
}
}
-
+
return mapping;
}
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java Fri May 2 12:34:23 2014
@@ -30,7 +30,7 @@ import opennlp.tools.util.Span;
public class InSpanGenerator extends FeatureGeneratorAdapter {
private final String prefix;
-
+
private final TokenNameFinder finder;
private String currentSentence[];
@@ -38,22 +38,22 @@ public class InSpanGenerator extends Fea
private Span currentNames[];
/**
- * Initializes the current instance.
+ * Initializes the current instance.
*
* @param prefix the prefix is used to distinguish the generated features
* from features generated by other instances of {@link InSpanGenerator}s.
* @param finder the {@link TokenNameFinder} used to detect the names.
*/
public InSpanGenerator(String prefix, TokenNameFinder finder) {
-
- if (prefix == null)
+
+ if (prefix == null)
throw new IllegalArgumentException("prefix must not be null!");
-
+
this.prefix = prefix;
-
+
if (finder == null)
throw new IllegalArgumentException("finder must not be null!");
-
+
this.finder = finder;
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java Fri May 2 12:34:23 2014
@@ -22,7 +22,7 @@ import java.util.List;
public class PrefixFeatureGenerator extends FeatureGeneratorAdapter {
private static final int PREFIX_LENGTH = 4;
-
+
public static String[] getPrefixes(String lex) {
String[] prefs = new String[PREFIX_LENGTH];
for (int li = 0, ll = PREFIX_LENGTH; li < ll; li++) {
@@ -30,7 +30,7 @@ public class PrefixFeatureGenerator exte
}
return prefs;
}
-
+
public void createFeatures(List<String> features, String[] tokens, int index,
String[] previousOutcomes) {
String[] prefs = PrefixFeatureGenerator.getPrefixes(tokens[index]);
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java Fri May 2 12:34:23 2014
@@ -26,13 +26,13 @@ public class SentenceFeatureGenerator ex
private final boolean isGenerateFirstWordFeature;
private final boolean isGenerateLastWordFeature;
-
+
public SentenceFeatureGenerator(boolean isGenerateFirstWordFeature,
boolean isGenerateLastWordFeature) {
this.isGenerateFirstWordFeature = isGenerateFirstWordFeature;
this.isGenerateLastWordFeature = isGenerateLastWordFeature;
}
-
+
public void createFeatures(List<String> features, String[] tokens, int index,
String[] previousOutcomes) {
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java Fri May 2 12:34:23 2014
@@ -38,7 +38,7 @@ public class StringPattern {
private final int pattern;
private final int digits;
-
+
private StringPattern(int pattern, int digits) {
this.pattern = pattern;
this.digits = digits;
@@ -50,7 +50,7 @@ public class StringPattern {
public boolean isAllLetter() {
return (pattern & ALL_LETTERS) > 0;
}
-
+
/**
* @return true if first letter is capital.
*/
@@ -64,14 +64,14 @@ public class StringPattern {
public boolean isAllCapitalLetter() {
return (pattern & ALL_CAPITAL_LETTER) > 0;
}
-
+
/**
* @return true if all letters are lower case.
*/
public boolean isAllLowerCaseLetter() {
return (pattern & ALL_LOWERCASE_LETTER) > 0;
}
-
+
/**
* @return true if all chars are digits.
*/
@@ -85,7 +85,7 @@ public class StringPattern {
public int digits() {
return digits;
}
-
+
public boolean containsPeriod() {
return (pattern & CONTAINS_PERIOD) > 0;
}
@@ -113,9 +113,9 @@ public class StringPattern {
public static StringPattern recognize(String token) {
int pattern = ALL_CAPITAL_LETTER | ALL_LOWERCASE_LETTER | ALL_DIGIT | ALL_LETTERS;
-
+
int digits = 0;
-
+
for (int i = 0; i < token.length(); i++) {
final char ch = token.charAt(i);
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java Fri May 2 12:34:23 2014
@@ -22,7 +22,7 @@ import java.util.List;
public class SuffixFeatureGenerator extends FeatureGeneratorAdapter {
private static final int SUFFIX_LENGTH = 4;
-
+
public static String[] getSuffixes(String lex) {
String[] suffs = new String[SUFFIX_LENGTH];
for (int li = 0, ll = SUFFIX_LENGTH; li < ll; li++) {
@@ -30,7 +30,7 @@ public class SuffixFeatureGenerator exte
}
return suffs;
}
-
+
public void createFeatures(List<String> features, String[] tokens, int index,
String[] previousOutcomes) {
String[] suffs = SuffixFeatureGenerator.getSuffixes(tokens[index]);
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/W2VClassesDictionary.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/W2VClassesDictionary.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/W2VClassesDictionary.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/W2VClassesDictionary.java Fri May 2 12:34:23 2014
@@ -47,7 +47,7 @@ public class W2VClassesDictionary implem
artifact.serialize(out);
}
}
-
+
private Map<String, String> tokenToClusterMap = new HashMap<String, String>();
public W2VClassesDictionary(InputStream in) throws IOException {
@@ -70,11 +70,11 @@ public class W2VClassesDictionary implem
public void serialize(OutputStream out) throws IOException {
Writer writer = new BufferedWriter(new OutputStreamWriter(out));
-
+
for (Map.Entry<String, String> entry : tokenToClusterMap.entrySet()) {
writer.write(entry.getKey() + " " + entry.getValue() + "\n");
}
-
+
writer.flush();
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WindowFeatureGenerator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WindowFeatureGenerator.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WindowFeatureGenerator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WindowFeatureGenerator.java Fri May 2 12:34:23 2014
@@ -52,10 +52,10 @@ public class WindowFeatureGenerator impl
this.prevWindowSize = prevWindowSize;
this.nextWindowSize = nextWindowSize;
}
-
+
/**
* Initializes the current instance with the given parameters.
- *
+ *
* @param prevWindowSize
* @param nextWindowSize
* @param generators
@@ -63,7 +63,7 @@ public class WindowFeatureGenerator impl
public WindowFeatureGenerator(int prevWindowSize, int nextWindowSize, AdaptiveFeatureGenerator... generators) {
this(new AggregatedFeatureGenerator(generators), prevWindowSize, nextWindowSize);
}
-
+
/**
* Initializes the current instance. The previous and next window size is 5.
*
@@ -72,16 +72,16 @@ public class WindowFeatureGenerator impl
public WindowFeatureGenerator(AdaptiveFeatureGenerator generator) {
this(generator, 5, 5);
}
-
+
/**
* Initializes the current instance with the given parameters.
- *
+ *
* @param generators array of feature generators
*/
public WindowFeatureGenerator(AdaptiveFeatureGenerator... generators) {
this(new AggregatedFeatureGenerator(generators), 5, 5);
}
-
+
public void createFeatures(List<String> features, String[] tokens, int index, String[] preds) {
// current features
generator.createFeatures(features, tokens, index, preds);
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ArtifactProvider.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ArtifactProvider.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ArtifactProvider.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ArtifactProvider.java Fri May 2 12:34:23 2014
@@ -26,7 +26,7 @@ public interface ArtifactProvider {
* Gets an artifact by name
*/
public <T> T getArtifact(String key);
-
+
/**
* Retrieves the value to the given key from the manifest.properties
* entry.
@@ -40,16 +40,16 @@ public interface ArtifactProvider {
/**
* Retrieves the language code of the material which was used to train the
* model or x-unspecified if non was set.
- *
+ *
* @return the language code of this model
*/
public String getLanguage();
-
+
/**
* Indicates if this provider was loaded from serialized. It is useful, for
* example, while validating artifacts: you can skip the time consuming ones
* if they where already validated during the serialization.
- *
+ *
* @return true if this model was loaded from serialized
*/
public boolean isLoadedFromSerialized();
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java Fri May 2 12:34:23 2014
@@ -41,59 +41,59 @@ import opennlp.tools.util.ext.ExtensionL
/**
* This model is a common based which can be used by the components
* model classes.
- *
+ *
* TODO:
* Provide sub classes access to serializers already in constructor
*/
public abstract class BaseModel implements ArtifactProvider {
private static int MODEL_BUFFER_SIZE_LIMIT = Integer.MAX_VALUE;
-
+
protected static final String MANIFEST_ENTRY = "manifest.properties";
protected static final String FACTORY_NAME = "factory";
-
+
private static final String MANIFEST_VERSION_PROPERTY = "Manifest-Version";
private static final String COMPONENT_NAME_PROPERTY = "Component-Name";
private static final String VERSION_PROPERTY = "OpenNLP-Version";
private static final String TIMESTAMP_PROPERTY = "Timestamp";
private static final String LANGUAGE_PROPERTY = "Language";
-
+
public static final String TRAINING_CUTOFF_PROPERTY = "Training-Cutoff";
public static final String TRAINING_ITERATIONS_PROPERTY = "Training-Iterations";
public static final String TRAINING_EVENTHASH_PROPERTY = "Training-Eventhash";
-
+
private static String SERIALIZER_CLASS_NAME_PREFIX = "serializer-class-";
-
+
private Map<String, ArtifactSerializer> artifactSerializers =
new HashMap<String, ArtifactSerializer>();
protected final Map<String, Object> artifactMap = new HashMap<String, Object>();
-
+
protected BaseToolFactory toolFactory;
-
+
private final String componentName;
private boolean subclassSerializersInitiated = false;
private boolean finishedLoadingArtifacts = false;
-
+
private final boolean isLoadedFromSerialized;
private BaseModel(String componentName, boolean isLoadedFromSerialized) {
this.isLoadedFromSerialized = isLoadedFromSerialized;
-
+
if (componentName == null)
throw new IllegalArgumentException("componentName must not be null!");
-
+
this.componentName = componentName;
}
-
+
/**
* Initializes the current instance. The sub-class constructor should call the
* method {@link #checkArtifactMap()} to check the artifact map is OK.
* <p>
* Sub-classes will have access to custom artifacts and serializers provided
* by the factory.
- *
+ *
* @param componentName
* the component name
* @param languageCode
@@ -112,24 +112,24 @@ public abstract class BaseModel implemen
throw new IllegalArgumentException("languageCode must not be null!");
createBaseArtifactSerializers(artifactSerializers);
-
+
Properties manifest = new Properties();
manifest.setProperty(MANIFEST_VERSION_PROPERTY, "1.0");
manifest.setProperty(LANGUAGE_PROPERTY, languageCode);
manifest.setProperty(VERSION_PROPERTY, Version.currentVersion().toString());
- manifest.setProperty(TIMESTAMP_PROPERTY,
+ manifest.setProperty(TIMESTAMP_PROPERTY,
Long.toString(System.currentTimeMillis()));
manifest.setProperty(COMPONENT_NAME_PROPERTY, componentName);
-
+
if (manifestInfoEntries != null) {
for (Map.Entry<String, String> entry : manifestInfoEntries.entrySet()) {
manifest.setProperty(entry.getKey(), entry.getValue());
}
}
-
+
artifactMap.put(MANIFEST_ENTRY, manifest);
finishedLoadingArtifacts = true;
-
+
if (factory!=null) {
setManifestProperty(FACTORY_NAME, factory.getClass().getCanonicalName());
artifactMap.putAll(factory.createArtifactMap());
@@ -140,7 +140,7 @@ public abstract class BaseModel implemen
setManifestProperty(key, entries.get(key));
}
}
-
+
try {
initializeFactory();
} catch (InvalidFormatException e) {
@@ -152,7 +152,7 @@ public abstract class BaseModel implemen
/**
* Initializes the current instance. The sub-class constructor should call the
* method {@link #checkArtifactMap()} to check the artifact map is OK.
- *
+ *
* @param componentName
* the component name
* @param languageCode
@@ -163,10 +163,10 @@ public abstract class BaseModel implemen
protected BaseModel(String componentName, String languageCode, Map<String, String> manifestInfoEntries) {
this(componentName, languageCode, manifestInfoEntries, null);
}
-
+
/**
* Initializes the current instance.
- *
+ *
* @param componentName the component name
* @param in the input stream containing the model
*
@@ -175,15 +175,15 @@ public abstract class BaseModel implemen
*/
protected BaseModel(String componentName, InputStream in) throws IOException, InvalidFormatException {
this(componentName, true);
-
+
loadModel(in);
}
protected BaseModel(String componentName, File modelFile) throws IOException, InvalidFormatException {
this(componentName, true);
-
+
InputStream in = new BufferedInputStream(new FileInputStream(modelFile));
-
+
try {
loadModel(in);
}
@@ -194,7 +194,7 @@ public abstract class BaseModel implemen
protected BaseModel(String componentName, URL modelURL) throws IOException, InvalidFormatException {
this(componentName, true);
-
+
InputStream in = new BufferedInputStream(modelURL.openStream());
try {
@@ -206,32 +206,32 @@ public abstract class BaseModel implemen
}
private void loadModel(InputStream in) throws IOException, InvalidFormatException {
-
+
if (in == null) {
throw new IllegalArgumentException("in must not be null!");
}
-
+
createBaseArtifactSerializers(artifactSerializers);
if (!in.markSupported()) {
in = new BufferedInputStream(in);
}
-
- // TODO: Discuss this solution, the buffering should
+
+ // TODO: Discuss this solution, the buffering should
in.mark(MODEL_BUFFER_SIZE_LIMIT);
-
+
final ZipInputStream zip = new ZipInputStream(in);
-
+
// The model package can contain artifacts which are serialized with 3rd party
// serializers which are configured in the manifest file. To be able to load
- // the model the manifest must be read first, and afterwards all the artifacts
+ // the model the manifest must be read first, and afterwards all the artifacts
// can be de-serialized.
-
+
// The ordering of artifacts in a zip package is not guaranteed. The stream is first
// read until the manifest appears, reseted, and read again to load all artifacts.
-
+
boolean isSearchingForManifest = true;
-
+
ZipEntry entry;
while((entry = zip.getNextEntry()) != null && isSearchingForManifest) {
@@ -244,20 +244,20 @@ public abstract class BaseModel implemen
zip.closeEntry();
}
-
+
initializeFactory();
-
+
loadArtifactSerializers();
// The Input Stream should always be reset-able because if markSupport returns
// false it is wrapped before hand into an Buffered InputStream
in.reset();
-
+
finishLoadingArtifacts(in);
-
+
checkArtifactMap();
}
-
+
private void initializeFactory() throws InvalidFormatException {
String factoryName = getManifestProperty(FACTORY_NAME);
if (factoryName == null) {
@@ -274,17 +274,17 @@ public abstract class BaseModel implemen
}
}
}
-
+
/**
* Sub-classes should override this method if their module has a default
* BaseToolFactory sub-class.
- *
+ *
* @return the default {@link BaseToolFactory} for the module, or null if none.
*/
protected Class<? extends BaseToolFactory> getDefaultFactory() {
return null;
}
-
+
/**
* Loads the artifact serializers.
*/
@@ -299,23 +299,23 @@ public abstract class BaseModel implemen
*/
private void finishLoadingArtifacts(InputStream in)
throws InvalidFormatException, IOException {
-
+
final ZipInputStream zip = new ZipInputStream(in);
-
+
Map<String, Object> artifactMap = new HashMap<String, Object>();
-
+
ZipEntry entry;
while((entry = zip.getNextEntry()) != null ) {
-
+
// Note: The manifest.properties file will be read here again,
// there should be no need to prevent that.
-
+
String entryName = entry.getName();
String extension = getEntryExtension(entryName);
ArtifactSerializer factory = artifactSerializers.get(extension);
- String artifactSerializerClazzName =
+ String artifactSerializerClazzName =
getManifestProperty(SERIALIZER_CLASS_NAME_PREFIX + entryName);
if (artifactSerializerClazzName != null) {
@@ -323,18 +323,18 @@ public abstract class BaseModel implemen
factory = ExtensionLoader.instantiateExtension(ArtifactSerializer.class, artifactSerializerClazzName);
}
}
-
+
if (factory != null) {
artifactMap.put(entryName, factory.create(zip));
} else {
throw new InvalidFormatException("Unknown artifact format: " + extension);
}
-
+
zip.closeEntry();
}
this.artifactMap.putAll(artifactMap);
-
+
finishedLoadingArtifacts = true;
}
@@ -364,30 +364,30 @@ public abstract class BaseModel implemen
throw new IllegalStateException(e);
}
- return artifactSerializers.get(extension);
+ return artifactSerializers.get(extension);
}
-
+
protected static Map<String, ArtifactSerializer> createArtifactSerializers() {
Map<String, ArtifactSerializer> serializers = new HashMap<String, ArtifactSerializer>();
-
+
GenericModelSerializer.register(serializers);
PropertiesSerializer.register(serializers);
DictionarySerializer.register(serializers);
-
+
return serializers;
}
-
+
/**
* Registers all {@link ArtifactSerializer} for their artifact file name extensions.
* The registered {@link ArtifactSerializer} are used to create and serialize
* resources in the model package.
- *
+ *
* Override this method to register custom {@link ArtifactSerializer}s.
*
* Note:
* Subclasses should generally invoke super.createArtifactSerializers at the beginning
* of this method.
- *
+ *
* This method is called during construction.
*
* @param serializers the key of the map is the file extension used to lookup
@@ -403,7 +403,7 @@ public abstract class BaseModel implemen
Map<String, ArtifactSerializer> serializers) {
serializers.putAll(createArtifactSerializers());
}
-
+
/**
* Validates the parsed artifacts. If something is not
* valid subclasses should throw an {@link InvalidFormatException}.
@@ -420,29 +420,29 @@ public abstract class BaseModel implemen
// First check version, everything else might change in the future
String versionString = getManifestProperty(VERSION_PROPERTY);
-
+
if (versionString != null) {
Version version;
-
+
try {
version = Version.parse(versionString);
}
catch (NumberFormatException e) {
throw new InvalidFormatException("Unable to parse model version '" + versionString + "'!", e);
}
-
+
// Version check is only performed if current version is not the dev/debug version
if (!Version.currentVersion().equals(Version.DEV_VERSION)) {
- // Major and minor version must match, revision might be
+ // Major and minor version must match, revision might be
if (Version.currentVersion().getMajor() != version.getMajor() ||
Version.currentVersion().getMinor() != version.getMinor()) {
//this check allows for the use of models one minor release behind current minor release
if(Version.currentVersion().getMajor() == version.getMajor() && (Version.currentVersion().getMinor()-1) != version.getMinor()){
- throw new InvalidFormatException("Model version " + version + " is not supported by this ("
+ throw new InvalidFormatException("Model version " + version + " is not supported by this ("
+ Version.currentVersion() +") version of OpenNLP!");
}
}
-
+
// Reject loading a snapshot model with a non-snapshot version
if (!Version.currentVersion().isSnapshot() && version.isSnapshot()) {
throw new InvalidFormatException("Model version " + version + " is a snapshot - snapshot models are not " +
@@ -454,21 +454,21 @@ public abstract class BaseModel implemen
throw new InvalidFormatException("Missing " + VERSION_PROPERTY + " property in " +
MANIFEST_ENTRY + "!");
}
-
+
if (getManifestProperty(COMPONENT_NAME_PROPERTY) == null)
throw new InvalidFormatException("Missing " + COMPONENT_NAME_PROPERTY + " property in " +
MANIFEST_ENTRY + "!");
-
- if (!getManifestProperty(COMPONENT_NAME_PROPERTY).equals(componentName))
- throw new InvalidFormatException("The " + componentName + " cannot load a model for the " +
+
+ if (!getManifestProperty(COMPONENT_NAME_PROPERTY).equals(componentName))
+ throw new InvalidFormatException("The " + componentName + " cannot load a model for the " +
getManifestProperty(COMPONENT_NAME_PROPERTY) + "!");
-
+
if (getManifestProperty(LANGUAGE_PROPERTY) == null)
throw new InvalidFormatException("Missing " + LANGUAGE_PROPERTY + " property in " +
MANIFEST_ENTRY + "!");
-
+
// Validate the factory. We try to load it using the ExtensionLoader. It
- // will return the factory, null or raise an exception
+ // will return the factory, null or raise an exception
String factoryName = getManifestProperty(FACTORY_NAME);
if (factoryName != null) {
try {
@@ -484,7 +484,7 @@ public abstract class BaseModel implemen
+ factoryName, e);
}
}
-
+
// validate artifacts declared by the factory
if(toolFactory != null) {
toolFactory.validateArtifactMap();
@@ -492,7 +492,7 @@ public abstract class BaseModel implemen
}
/**
- * Checks the artifact map.
+ * Checks the artifact map.
* <p>
* A subclass should call this method from a constructor which accepts the individual
* artifact map items, to validate that these items form a valid model.
@@ -509,7 +509,7 @@ public abstract class BaseModel implemen
throw new IllegalArgumentException(e);
}
}
-
+
/**
* Retrieves the value to the given key from the manifest.properties
* entry.
@@ -558,7 +558,7 @@ public abstract class BaseModel implemen
return Version.parse(version);
}
-
+
/**
* Serializes the model to the given {@link OutputStream}.
*
@@ -585,40 +585,40 @@ public abstract class BaseModel implemen
artifactSerializerName);
}
}
-
+
ZipOutputStream zip = new ZipOutputStream(out);
for (String name : artifactMap.keySet()) {
zip.putNextEntry(new ZipEntry(name));
Object artifact = artifactMap.get(name);
-
+
ArtifactSerializer serializer = getArtifactSerializer(name);
// If model is serialize-able always use the provided serializer
if (artifact instanceof SerializableArtifact) {
-
+
SerializableArtifact serializableArtifact = (SerializableArtifact) artifact;
String artifactSerializerName =
serializableArtifact.getArtifactSerializerClass().getName();
-
+
serializer = ExtensionLoader.instantiateExtension(ArtifactSerializer.class, artifactSerializerName);
}
-
+
if (serializer == null) {
throw new IllegalStateException("Missing serializer for " + name);
}
-
+
serializer.serialize(artifactMap.get(name), zip);
zip.closeEntry();
}
-
+
zip.finish();
zip.flush();
}
-
+
@SuppressWarnings("unchecked")
public <T> T getArtifact(String key) {
Object artifact = artifactMap.get(key);
@@ -626,7 +626,7 @@ public abstract class BaseModel implemen
return null;
return (T) artifact;
}
-
+
private static byte[] toByteArray(InputStream input) throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[1024 * 4];
@@ -642,27 +642,27 @@ public abstract class BaseModel implemen
public boolean isLoadedFromSerialized() {
return isLoadedFromSerialized;
}
-
+
public static void main(String[] args) throws Exception {
-
- // create a stream which can be reset, enclose it in a buffered stream which supports reseting
+
+ // create a stream which can be reset, enclose it in a buffered stream which supports reseting
InputStream in = new FileInputStream("annotation.conf");
-
+
System.out.println("Is mark supported: " + in.markSupported());
-
+
in = new BufferedInputStream(in);
-
+
System.out.println("Is mark supported: " + in.markSupported());
-
- // 2 GB limit
+
+ // 2 GB limit
in.mark(4096);
-
+
in.read();
-
+
in.reset();
-
+
// the mark support can be used to test if reseting is supported, we shoudl use this test anyway
// to fail gracefully in the cross validators ...
-
+
}
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ClassSerializer.java Fri May 2 12:34:23 2014
@@ -28,16 +28,16 @@ import opennlp.tools.util.InvalidFormatE
public class ClassSerializer implements ArtifactSerializer<Class<?>> {
private static final String CLASS_SEARCH_NAME = "ClassSearchName";
-
+
private byte[] classBytes;
-
+
private static Class<?> loadClass(final byte[] classBytes)
throws InvalidFormatException {
ClassLoader loader = new ClassLoader() {
@Override
protected Class<?> findClass(String name) throws ClassNotFoundException {
- if (CLASS_SEARCH_NAME.equals(name))
+ if (CLASS_SEARCH_NAME.equals(name))
return defineClass(null, classBytes, 0, classBytes.length);
else
return super.findClass(name);
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/FeatureGeneratorFactorySerializer.java Fri May 2 12:34:23 2014
@@ -26,20 +26,20 @@ import opennlp.tools.util.InvalidFormatE
import opennlp.tools.util.featuregen.FeatureGeneratorFactory;
@Deprecated
-public class FeatureGeneratorFactorySerializer
+public class FeatureGeneratorFactorySerializer
implements ArtifactSerializer<FeatureGeneratorFactory>{
private ClassSerializer classSerializer;
-
+
public FeatureGeneratorFactorySerializer() {
classSerializer = new ClassSerializer();
}
public FeatureGeneratorFactory create(InputStream in) throws IOException,
InvalidFormatException {
-
+
Class<?> generatorFactoryClass = classSerializer.create(in);
-
+
try {
return (FeatureGeneratorFactory) generatorFactoryClass.newInstance();
} catch (InstantiationException e) {
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ModelUtil.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ModelUtil.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ModelUtil.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/ModelUtil.java Fri May 2 12:34:23 2014
@@ -50,19 +50,19 @@ public final class ModelUtil {
*
* @param model the model to be written
* @param out the stream the model should be written to
- *
+ *
* @throws IOException
* @throws IllegalArgumentException in case one of the parameters is null
*/
- public static void writeModel(MaxentModel model, final OutputStream out)
+ public static void writeModel(MaxentModel model, final OutputStream out)
throws IOException, IllegalArgumentException {
-
+
if (model == null)
throw new IllegalArgumentException("model parameter must not be null!");
-
+
if (out == null)
throw new IllegalArgumentException("out parameter must not be null!");
-
+
GenericModelWriter modelWriter = new GenericModelWriter((AbstractModel) model, new DataOutputStream(new OutputStream() {
@Override
public void write(int b) throws IOException {
@@ -102,14 +102,14 @@ public final class ModelUtil {
return result;
}
-
+
/**
* Writes the provided {@link InputStream} into a byte array
* which is returned
- *
+ *
* @param in stream to read data for the byte array from
* @return byte array with the contents of the stream
- *
+ *
* @throws IOException if an exception is thrown while reading
* from the provided {@link InputStream}
*/
@@ -125,19 +125,19 @@ public final class ModelUtil {
return byteArrayOut.toByteArray();
}
-
+
public static void addCutoffAndIterations(Map<String, String> manifestInfoEntries,
int cutoff, int iterations) {
manifestInfoEntries.put(BaseModel.TRAINING_CUTOFF_PROPERTY, Integer.toString(cutoff));
manifestInfoEntries.put(BaseModel.TRAINING_ITERATIONS_PROPERTY, Integer.toString(iterations));
}
-
+
/**
* Creates the default training parameters in case they are not provided.
- *
+ *
* Note: Do not use this method, internal use only!
- *
- *
+ *
+ *
* @return training parameters instance
*/
public static TrainingParameters createDefaultTrainingParameters() {
@@ -145,7 +145,7 @@ public final class ModelUtil {
mlParams.put(TrainingParameters.ALGORITHM_PARAM, GIS.MAXENT_VALUE);
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
mlParams.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(5));
-
+
return mlParams;
}
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/SerializableArtifact.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/SerializableArtifact.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/SerializableArtifact.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/model/SerializableArtifact.java Fri May 2 12:34:23 2014
@@ -18,14 +18,14 @@
package opennlp.tools.util.model;
public interface SerializableArtifact {
-
+
/**
* Retrieves the class which can serialize and recreate this artifact.
* <br>
- * Note:
- * The serializer class must have a public zero argument constructor or
+ * Note:
+ * The serializer class must have a public zero argument constructor or
* an exception is thrown during model serialization/loading.
- *
+ *
* @return the corresponding ArtifactSerializer class.
*/
Class<?> getArtifactSerializerClass();
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -32,9 +32,9 @@ public class ChunkSampleStreamTest{
@Test
public void testReadingEvents() throws IOException {
-
+
StringBuilder sample = new StringBuilder();
-
+
// First sample sentence
sample.append("word11 tag11 pred11");
sample.append('\n');
@@ -42,10 +42,10 @@ public class ChunkSampleStreamTest{
sample.append('\n');
sample.append("word13 tag13 pred13");
sample.append('\n');
-
+
// Start next sample sentence
sample.append('\n');
-
+
// Second sample sentence
sample.append("word21 tag21 pred21");
sample.append('\n');
@@ -53,11 +53,11 @@ public class ChunkSampleStreamTest{
sample.append('\n');
sample.append("word23 tag23 pred23");
sample.append('\n');
-
+
ObjectStream<String> stringStream = new PlainTextByLineStream(new StringReader(sample.toString()));
-
+
ObjectStream<ChunkSample> chunkStream = new ChunkSampleStream(stringStream);
-
+
// read first sample
ChunkSample firstSample = chunkStream.read();
assertEquals("word11", firstSample.getSentence()[0]);
@@ -69,8 +69,8 @@ public class ChunkSampleStreamTest{
assertEquals("word13", firstSample.getSentence()[2]);
assertEquals("tag13", firstSample.getTags()[2]);
assertEquals("pred13", firstSample.getPreds()[2]);
-
-
+
+
// read second sample
ChunkSample secondSample = chunkStream.read();
assertEquals("word21", secondSample.getSentence()[0]);
@@ -82,7 +82,7 @@ public class ChunkSampleStreamTest{
assertEquals("word23", secondSample.getSentence()[2]);
assertEquals("tag23", secondSample.getTags()[2]);
assertEquals("pred23", secondSample.getPreds()[2]);
-
+
assertNull(chunkStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkSampleTest.java Fri May 2 12:34:23 2014
@@ -41,7 +41,7 @@ public class ChunkSampleTest {
new ChunkSample(new String[]{""}, new String[]{""},
new String[]{"test", "one element to much"});
}
-
+
private static String[] createSentence() {
return new String[] {
"Forecasts",
@@ -62,9 +62,9 @@ public class ChunkSampleTest {
"."
};
}
-
+
private static String[] createTags() {
-
+
return new String[]{
"NNS",
"IN",
@@ -84,7 +84,7 @@ public class ChunkSampleTest {
"."
};
}
-
+
private static String[] createChunks() {
return new String[]{
"B-NP",
@@ -105,24 +105,24 @@ public class ChunkSampleTest {
"O"
};
}
-
+
@Test
public void testRetrievingContent() {
ChunkSample sample = new ChunkSample(createSentence(), createTags(), createChunks());
-
+
assertArrayEquals(createSentence(), sample.getSentence());
assertArrayEquals(createTags(), sample.getTags());
assertArrayEquals(createChunks(), sample.getPreds());
}
-
+
@Test
public void testToString() throws IOException {
-
+
ChunkSample sample = new ChunkSample(createSentence(), createTags(), createChunks());
String[] sentence = createSentence();
String[] tags = createTags();
String[] chunks = createChunks();
-
+
StringReader sr = new StringReader(sample.toString());
BufferedReader reader = new BufferedReader(sr);
for (int i = 0; i < sentence.length; i++) {
@@ -134,17 +134,17 @@ public class ChunkSampleTest {
assertEquals(chunks[i], parts[2]);
}
}
-
+
@Test
public void testNicePrint() {
-
+
ChunkSample sample = new ChunkSample(createSentence(), createTags(), createChunks());
-
+
assertEquals(" [NP Forecasts_NNS ] [PP for_IN ] [NP the_DT trade_NN figures_NNS ] " +
"[VP range_VBP ] [ADVP widely_RB ] ,_, [NP Forecasts_NNS ] [PP for_IN ] [NP the_DT trade_NN figures_NNS ] " +
"[VP range_VBP ] [ADVP widely_RB ] ._.", sample.nicePrint());
}
-
+
@Test
public void testAsSpan() {
ChunkSample sample = new ChunkSample(createSentence(), createTags(),
@@ -163,7 +163,7 @@ public class ChunkSampleTest {
assertEquals(new Span(13, 14, "VP"), spans[8]);
assertEquals(new Span(14, 15, "ADVP"), spans[9]);
}
-
+
@Test
public void testPhraseAsSpan() {
Span[] spans = ChunkSample.phrasesAsSpanList(createSentence(),
@@ -196,11 +196,11 @@ public class ChunkSampleTest {
ChunkSample cs1 = predictedSample.read();
String[] g1 = Span.spansToStrings(cs1.getPhrasesAsSpanList(), cs1.getSentence());
assertEquals(15, g1.length);
-
+
ChunkSample cs2 = predictedSample.read();
String[] g2 = Span.spansToStrings(cs2.getPhrasesAsSpanList(), cs2.getSentence());
assertEquals(10, g2.length);
-
+
ChunkSample cs3 = predictedSample.read();
String[] g3 = Span.spansToStrings(cs3.getPhrasesAsSpanList(), cs3.getSentence());
assertEquals(7, g3.length);
@@ -212,7 +212,7 @@ public class ChunkSampleTest {
assertEquals("lifetime access", g3[5]);
assertEquals("to", g3[6]);
}
-
+
// following are some tests to check the argument validation. Since all uses
// the same validateArguments method, we do a deeper test only once
@@ -242,7 +242,7 @@ public class ChunkSampleTest {
new ChunkSample(Arrays.asList(new String[1]), Arrays.asList(new String[1]),
Arrays.asList(new String[2]));
}
-
+
@Test
public void testEquals() {
assertFalse(createGoldSample() == createGoldSample());
@@ -250,15 +250,15 @@ public class ChunkSampleTest {
assertFalse(createPredSample().equals(createGoldSample()));
assertFalse(createPredSample().equals(new Object()));
}
-
+
public static ChunkSample createGoldSample() {
return new ChunkSample(createSentence(), createTags(), createChunks());
}
-
+
public static ChunkSample createPredSample() {
String[] chunks = createChunks();
chunks[5] = "B-NP";
return new ChunkSample(createSentence(), createTags(), chunks);
}
-
+
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java Fri May 2 12:34:23 2014
@@ -38,7 +38,7 @@ public class ChunkerDetailedFMeasureList
"opennlp/tools/chunker/output.txt");
InputStream inExpected = getClass().getClassLoader().getResourceAsStream(
"opennlp/tools/chunker/output.txt");
-
+
InputStream detailedOutputStream = getClass().getClassLoader().getResourceAsStream(
"opennlp/tools/chunker/detailedOutput.txt");
@@ -62,7 +62,7 @@ public class ChunkerDetailedFMeasureList
StringBuilder expected = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(detailedOutputStream, encoding));
String line = reader.readLine();
-
+
while(line != null ) {
expected.append(line);
expected.append("\n");
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java Fri May 2 12:34:23 2014
@@ -34,11 +34,11 @@ import org.junit.Test;
/**
* Tests for {@link ChunkerEvaluator}.
- *
+ *
* @see ChunkerEvaluator
*/
public class ChunkerEvaluatorTest {
-
+
private static final double DELTA = 1.0E-9d;
/**
@@ -59,27 +59,27 @@ public class ChunkerEvaluatorTest {
DummyChunkSampleStream predictedSample = new DummyChunkSampleStream(
new PlainTextByLineStream(new InputStreamReader(inPredicted, encoding)), true);
-
+
DummyChunkSampleStream expectedSample = new DummyChunkSampleStream(
new PlainTextByLineStream(new InputStreamReader(inExpected)), false);
-
+
Chunker dummyChunker = new DummyChunker(predictedSample);
-
+
OutputStream stream = new ByteArrayOutputStream();
ChunkerEvaluationMonitor listener = new ChunkEvaluationErrorListener(stream);
ChunkerEvaluator evaluator = new ChunkerEvaluator(dummyChunker, listener);
-
+
evaluator.evaluate(expectedSample);
-
+
FMeasure fm = evaluator.getFMeasure();
-
+
assertEquals(0.8d, fm.getPrecisionScore(), DELTA);
assertEquals(0.875d, fm.getRecallScore(), DELTA);
-
+
assertNotSame(stream.toString().length(), 0);
-
+
}
-
+
@Test
public void testEvaluatorNoError() throws IOException {
InputStream inPredicted = getClass().getClassLoader().getResourceAsStream(
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerFactoryTest.java Fri May 2 12:34:23 2014
@@ -48,13 +48,13 @@ public class ChunkerFactoryTest {
sentences));
return stream;
}
-
+
static ChunkerModel trainModel(ModelType type, ChunkerFactory factory)
throws IOException {
return ChunkerME.train("en", createSampleStream(),
TrainingParameters.defaultParams(), factory);
}
-
+
@Test
public void testDefaultFactory() throws IOException {
@@ -75,7 +75,7 @@ public class ChunkerFactoryTest {
assertTrue(factory.getSequenceValidator() instanceof DefaultChunkerSequenceValidator);
}
-
+
@Test
public void testDummyFactory() throws IOException {
@@ -85,8 +85,8 @@ public class ChunkerFactoryTest {
assertTrue(factory instanceof DummyChunkerFactory);
assertTrue(factory.getContextGenerator() instanceof DummyChunkerFactory.DummyContextGenerator);
assertTrue(factory.getSequenceValidator() instanceof DummyChunkerFactory.DummySequenceValidator);
-
-
+
+
ByteArrayOutputStream out = new ByteArrayOutputStream();
model.serialize(out);
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
@@ -96,19 +96,19 @@ public class ChunkerFactoryTest {
factory = (DummyChunkerFactory)fromSerialized.getFactory();
assertTrue(factory.getContextGenerator() instanceof DefaultChunkerContextGenerator);
assertTrue(factory.getSequenceValidator() instanceof DefaultChunkerSequenceValidator);
-
-
+
+
ChunkerME chunker = new ChunkerME(model);
-
+
String[] toks1 = { "Rockwell", "said", "the", "agreement", "calls", "for",
"it", "to", "supply", "200", "additional", "so-called", "shipsets",
"for", "the", "planes", "." };
String[] tags1 = { "NNP", "VBD", "DT", "NN", "VBZ", "IN", "PRP", "TO", "VB",
"CD", "JJ", "JJ", "NNS", "IN", "DT", "NNS", "." };
-
-
+
+
chunker.chunk(toks1, tags1);
-
+
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java Fri May 2 12:34:23 2014
@@ -83,7 +83,7 @@ public class ChunkerMETest {
TrainingParameters params = new TrainingParameters();
params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
-
+
ChunkerModel chunkerModel = ChunkerME.train("en", sampleStream, params, new ChunkerFactory());
this.chunker = new ChunkerME(chunkerModel);
@@ -139,7 +139,7 @@ public class ChunkerMETest {
assertEquals(Arrays.asList(expect1), preds[0].getOutcomes());
assertNotSame(Arrays.asList(expect1), preds[1].getOutcomes());
}
-
+
@Test
public void testTokenProbArray() throws Exception {
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkSampleStream.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkSampleStream.java Fri May 2 12:34:23 2014
@@ -46,7 +46,7 @@ public class DummyChunkSampleStream exte
/**
* Returns a pair representing the expected and the predicted at 0: the
* chunk tag according to the corpus at 1: the chunk tag predicted
- *
+ *
* @see opennlp.tools.util.ObjectStream#read()
*/
public ChunkSample read() throws IOException {
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkerFactory.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkerFactory.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkerFactory.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/DummyChunkerFactory.java Fri May 2 12:34:23 2014
@@ -20,7 +20,7 @@ package opennlp.tools.chunker;
import opennlp.tools.util.SequenceValidator;
public class DummyChunkerFactory extends ChunkerFactory {
-
+
public DummyChunkerFactory() {
}
@@ -33,7 +33,7 @@ public class DummyChunkerFactory extends
public SequenceValidator<String> getSequenceValidator() {
return new DummySequenceValidator();
}
-
+
static class DummyContextGenerator extends DefaultChunkerContextGenerator {
@Override
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/ArgumentParserTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/ArgumentParserTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/ArgumentParserTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/ArgumentParserTest.java Fri May 2 12:34:23 2014
@@ -34,35 +34,35 @@ public class ArgumentParserTest {
interface ZeroMethods {
}
-
+
@Test(expected = IllegalArgumentException.class)
public void testZeroMethods() {
ArgumentParser.createUsage(ZeroMethods.class);
}
-
+
interface InvalidMethodName {
String invalidMethodName();
}
-
+
@Test(expected = IllegalArgumentException.class)
public void testInvalidMethodName() {
ArgumentParser.createUsage(InvalidMethodName.class);
}
-
+
interface InvalidReturnType {
Exception getTest();
}
-
+
@Test(expected = IllegalArgumentException.class)
public void testInvalidReturnType() {
ArgumentParser.createUsage(InvalidReturnType.class);
}
-
+
interface SimpleArguments extends AllOptionalArguments {
-
+
@ParameterDescription(valueName = "charset", description = "a charset encoding")
String getEncoding();
-
+
@OptionalParameter
Integer getCutoff();
}
@@ -78,24 +78,24 @@ public class ArgumentParserTest {
Boolean getAlphaNumOpt();
}
-
+
@Test
public void testSimpleArguments() {
-
+
String argsString = "-encoding UTF-8 -alphaNumOpt false";
-
+
SimpleArguments args = ArgumentParser.parse(argsString.split(" "), SimpleArguments.class);
-
+
assertEquals("UTF-8", args.getEncoding());
assertEquals(Integer.valueOf(100), args.getIterations());
assertEquals(null, args.getCutoff());
assertEquals(false, args.getAlphaNumOpt());
}
-
+
@Test(expected = IllegalArgumentException.class)
public void testSimpleArgumentsMissingEncoding() {
String argsString = "-alphaNumOpt false";
-
+
assertFalse(ArgumentParser.validateArguments(argsString.split(" "), SimpleArguments.class));
ArgumentParser.parse(argsString.split(" "), SimpleArguments.class);
}
@@ -126,46 +126,46 @@ public class ArgumentParserTest {
@Test
public void testSimpleArgumentsUsage() {
-
- String arguments[] = new String[] {"-encoding charset",
- "[-iterations num]",
+
+ String arguments[] = new String[] {"-encoding charset",
+ "[-iterations num]",
"[-alphaNumOpt true|false]"};
-
+
String usage = ArgumentParser.createUsage(SimpleArguments.class);
-
+
int expectedLength = 2;
for (String arg : arguments) {
assertTrue(usage.contains(arg));
expectedLength += arg.length();
}
-
+
assertTrue(usage.contains("a charset encoding"));
-
+
assertTrue(expectedLength < usage.length());
}
-
+
interface ExtendsEncodingParameter extends EncodingParameter {
-
+
@ParameterDescription(valueName = "value")
String getSomething();
}
-
-
+
+
@Test
public void testDefaultEncodingParameter() {
-
+
String args[] = "-something aValue".split(" ");
assertTrue(ArgumentParser.validateArguments(args, ExtendsEncodingParameter.class));
-
+
ExtendsEncodingParameter params = ArgumentParser.parse(args, ExtendsEncodingParameter.class);
assertEquals(Charset.defaultCharset(), params.getEncoding());
-
+
}
-
+
@Test
public void testSetEncodingParameter() {
-
+
Collection<Charset> availableCharset = Charset.availableCharsets().values();
String notTheDefaultCharset = "UTF-8";
for (Charset charset : availableCharset) {
@@ -174,12 +174,12 @@ public class ArgumentParserTest {
break;
}
}
-
+
String args[] = ("-something aValue -encoding " + notTheDefaultCharset).split(" ");
assertTrue(ArgumentParser.validateArguments(args, ExtendsEncodingParameter.class));
-
+
ExtendsEncodingParameter params = ArgumentParser.parse(args, ExtendsEncodingParameter.class);
assertEquals(Charset.forName(notTheDefaultCharset), params.getEncoding());
-
+
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/CLITest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/CLITest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/CLITest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/cmdline/CLITest.java Fri May 2 12:34:23 2014
@@ -29,50 +29,50 @@ public class CLITest {
private static class ExitException extends SecurityException {
private final int status;
-
+
public ExitException(int status) {
this.status = status;
}
-
+
int status() {
return status;
}
}
-
+
/**
* A <code>SecurityManager</code> which prevents System.exit anything else is allowed.
*/
private static class NoExitSecurityManager extends SecurityManager {
-
+
@Override
public void checkPermission(Permission perm) {
}
-
+
@Override
public void checkPermission(Permission perm, Object context) {
}
-
+
@Override
public void checkExit(int status){
super.checkExit(status);
-
+
throw new ExitException(status);
}
}
-
+
private final SecurityManager originalSecurityManager = System.getSecurityManager();
-
+
@Before
public void installNoExitSecurityManager() {
System.setSecurityManager(new NoExitSecurityManager());
}
-
+
/**
* Ensure the main method does not fail to print help message.
*/
@Test
public void testMainHelpMessage() {
-
+
try {
CLI.main(new String[]{});
} catch (ExitException e) {
@@ -115,14 +115,14 @@ public class CLITest {
assertEquals(-1, e.status());
}
}
-
-
+
+
/**
* Ensure all tools do not fail printing help message;
*/
@Test
public void testHelpMessageOfTools() {
-
+
for (String toolName : CLI.getToolNames()) {
try {
CLI.main(new String[]{toolName, "help"});
@@ -131,10 +131,10 @@ public class CLITest {
}
}
}
-
+
@After
public void restoreSecurityManager() {
System.setSecurityManager(originalSecurityManager);
}
-
+
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseInsensitiveTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseInsensitiveTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseInsensitiveTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseInsensitiveTest.java Fri May 2 12:34:23 2014
@@ -37,7 +37,7 @@ public class DictionaryAsSetCaseInsensit
private Dictionary getDict() {
return new Dictionary(false);
}
-
+
private StringList asSL(String str) {
return new StringList(str);
}
@@ -54,12 +54,12 @@ public class DictionaryAsSetCaseInsensit
Dictionary dict = getDict();
dict.put(asSL(a));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(a));
assertFalse(set.contains(b));
-
+
assertTrue(set.contains(a.toUpperCase()));
}
@@ -76,13 +76,13 @@ public class DictionaryAsSetCaseInsensit
dict.put(asSL(a));
dict.put(asSL(a1));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(a));
assertEquals(1, set.size());
}
-
+
/**
* Tests set.
*/
@@ -96,7 +96,7 @@ public class DictionaryAsSetCaseInsensit
dict.put(asSL(a));
dict.put(asSL(a1));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(a));
@@ -114,18 +114,18 @@ public class DictionaryAsSetCaseInsensit
Dictionary dictA = getDict();
dictA.put(asSL(entry1));
dictA.put(asSL(entry2));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
dictB.put(asSL(entry1));
dictB.put(asSL(entry2));
-
+
Set<String> setB = dictB.asStringSet();
assertTrue(setA.equals(setB));
}
-
+
/**
* Tests for the {@link Dictionary#equals(Object)} method.
*/
@@ -135,13 +135,13 @@ public class DictionaryAsSetCaseInsensit
Dictionary dictA = getDict();
dictA.put(asSL("1a"));
dictA.put(asSL("1b"));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
dictB.put(asSL("1A"));
dictB.put(asSL("1B"));
-
+
Set<String> setB = dictB.asStringSet();
assertTrue(setA.equals(setB));
@@ -156,7 +156,7 @@ public class DictionaryAsSetCaseInsensit
Dictionary dictA = getDict();
dictA.put(asSL(entry1));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
@@ -166,7 +166,7 @@ public class DictionaryAsSetCaseInsensit
assertEquals(setA.hashCode(), setB.hashCode());
}
-
+
/**
* Tests the {@link Dictionary#hashCode()} method.
*/
@@ -176,12 +176,12 @@ public class DictionaryAsSetCaseInsensit
Dictionary dictA = getDict();
dictA.put(asSL(entry1));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
dictB.put(asSL(entry1.toUpperCase()));
-
+
Set<String> setB = dictB.asStringSet();
// TODO: should it be equal??
@@ -201,18 +201,18 @@ public class DictionaryAsSetCaseInsensit
Dictionary dict = getDict();
dict.put(asSL(entry1));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(entry2));
}
-
+
/**
* Tests the iterator implementation
*/
@Test
public void testIterator() {
-
+
String entry1 = "1a";
String entry2 = "1b";
@@ -221,16 +221,16 @@ public class DictionaryAsSetCaseInsensit
dictA.put(asSL(entry2));
dictA.put(asSL(entry1.toUpperCase()));
dictA.put(asSL(entry2.toUpperCase()));
-
+
Iterator<String> it = dictA.asStringSet().iterator();
List<String> elements = new ArrayList<String>();
while (it.hasNext()) {
elements.add(it.next());
}
-
+
assertEquals(2, elements.size());
assertTrue(elements.contains(entry1));
assertTrue(elements.contains(entry2));
-
+
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseSensitiveTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseSensitiveTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseSensitiveTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryAsSetCaseSensitiveTest.java Fri May 2 12:34:23 2014
@@ -37,7 +37,7 @@ public class DictionaryAsSetCaseSensitiv
private Dictionary getDict() {
return new Dictionary(true);
}
-
+
private StringList asSL(String str) {
return new StringList(str);
}
@@ -54,12 +54,12 @@ public class DictionaryAsSetCaseSensitiv
Dictionary dict = getDict();
dict.put(asSL(a));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(a));
assertFalse(set.contains(b));
-
+
assertFalse(set.contains(a.toUpperCase()));
}
@@ -76,13 +76,13 @@ public class DictionaryAsSetCaseSensitiv
dict.put(asSL(a));
dict.put(asSL(a1));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(a));
assertEquals(1, set.size());
}
-
+
/**
* Tests set.
*/
@@ -96,7 +96,7 @@ public class DictionaryAsSetCaseSensitiv
dict.put(asSL(a));
dict.put(asSL(a1));
-
+
Set<String> set = dict.asStringSet();
assertTrue(set.contains(a));
@@ -114,18 +114,18 @@ public class DictionaryAsSetCaseSensitiv
Dictionary dictA = getDict();
dictA.put(asSL(entry1));
dictA.put(asSL(entry2));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
dictB.put(asSL(entry1));
dictB.put(asSL(entry2));
-
+
Set<String> setB = dictB.asStringSet();
assertTrue(setA.equals(setB));
}
-
+
/**
* Tests for the {@link Dictionary#equals(Object)} method.
*/
@@ -135,13 +135,13 @@ public class DictionaryAsSetCaseSensitiv
Dictionary dictA = getDict();
dictA.put(asSL("1a"));
dictA.put(asSL("1b"));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
dictB.put(asSL("1A"));
dictB.put(asSL("1B"));
-
+
Set<String> setB = dictB.asStringSet();
// should fail in case sensitive dict
@@ -157,7 +157,7 @@ public class DictionaryAsSetCaseSensitiv
Dictionary dictA = getDict();
dictA.put(asSL(entry1));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
@@ -167,7 +167,7 @@ public class DictionaryAsSetCaseSensitiv
assertEquals(setA.hashCode(), setB.hashCode());
}
-
+
/**
* Tests the {@link Dictionary#hashCode()} method.
*/
@@ -177,12 +177,12 @@ public class DictionaryAsSetCaseSensitiv
Dictionary dictA = getDict();
dictA.put(asSL(entry1));
-
+
Set<String> setA = dictA.asStringSet();
Dictionary dictB = getDict();
dictB.put(asSL(entry1.toUpperCase()));
-
+
Set<String> setB = dictB.asStringSet();
// TODO: should it be equal??
@@ -202,19 +202,19 @@ public class DictionaryAsSetCaseSensitiv
Dictionary dict = getDict();
dict.put(asSL(entry1));
-
+
Set<String> set = dict.asStringSet();
// should return false because 1a != 1A in a case sensitive lookup
assertFalse(set.contains(entry2));
}
-
+
/**
* Tests the iterator implementation
*/
@Test
public void testIterator() {
-
+
String entry1 = "1a";
String entry2 = "1b";
@@ -223,18 +223,18 @@ public class DictionaryAsSetCaseSensitiv
dictA.put(asSL(entry2));
dictA.put(asSL(entry1.toUpperCase()));
dictA.put(asSL(entry2.toUpperCase()));
-
+
Iterator<String> it = dictA.asStringSet().iterator();
List<String> elements = new ArrayList<String>();
while (it.hasNext()) {
elements.add(it.next());
}
-
+
assertEquals(4, elements.size());
assertTrue(elements.contains(entry1));
assertTrue(elements.contains(entry2));
assertTrue(elements.contains(entry1.toUpperCase()));
assertTrue(elements.contains(entry2.toUpperCase()));
-
+
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/dictionary/DictionaryTest.java Fri May 2 12:34:23 2014
@@ -42,14 +42,14 @@ public class DictionaryTest {
private Dictionary getCaseSensitive() {
return new Dictionary(true);
}
-
+
/**
* @return a case insensitive Dictionary
*/
private Dictionary getCaseInsensitive() {
return new Dictionary(false);
}
-
+
/**
* Tests a basic lookup.
*/
@@ -86,7 +86,7 @@ public class DictionaryTest {
assertTrue(!dict.contains(entry1u));
assertTrue(!dict.contains(entry2));
}
-
+
/**
* Tests serialization and deserailization of the {@link Dictionary}.
*
@@ -158,7 +158,7 @@ public class DictionaryTest {
Dictionary dictB = getCaseInsensitive();
dictB.put(entry1);
dictB.put(entry2);
-
+
Dictionary dictC = getCaseSensitive();
dictC.put(entry1);
dictC.put(entry2);
@@ -181,10 +181,10 @@ public class DictionaryTest {
Dictionary dictB = getCaseInsensitive();
dictB.put(entry2);
-
+
Dictionary dictC = getCaseSensitive();
dictC.put(entry1);
-
+
Dictionary dictD = getCaseSensitive();
dictD.put(entry2);
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentSampleTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentSampleTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentSampleTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentSampleTest.java Fri May 2 12:34:23 2014
@@ -24,7 +24,7 @@ import org.junit.Test;
public class DocumentSampleTest {
-
+
@Test
public void testEquals() {
assertFalse(createGoldSample() == createGoldSample());
@@ -40,5 +40,5 @@ public class DocumentSampleTest {
public static DocumentSample createPredSample() {
return new DocumentSample("anotherCategory", "a small text");
}
-
+
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll02NameSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -34,66 +34,66 @@ import opennlp.tools.util.Span;
import org.junit.Test;
/**
- *
+ *
* Note:
* Sample training data must be UTF-8 encoded and uncompressed!
*/
public class Conll02NameSampleStreamTest {
-
+
private static ObjectStream<NameSample> openData(LANGUAGE lang, String name) throws IOException {
- InputStreamFactory in = new ResourceAsStreamFactory(Conll02NameSampleStreamTest.class,
+ InputStreamFactory in = new ResourceAsStreamFactory(Conll02NameSampleStreamTest.class,
"/opennlp/tools/formats/" + name);
-
+
return new Conll02NameSampleStream(lang, in, Conll02NameSampleStream.GENERATE_PERSON_ENTITIES);
}
-
+
@Test
public void testParsingSpanishSample() throws IOException {
-
+
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.ES, "conll2002-es.sample");
-
+
NameSample personName = sampleStream.read();
-
+
assertNotNull(personName);
-
+
assertEquals(5, personName.getSentence().length);
assertEquals(1, personName.getNames().length);
assertEquals(true, personName.isClearAdaptiveDataSet());
-
+
Span nameSpan = personName.getNames()[0];
assertEquals(0, nameSpan.getStart());
assertEquals(4, nameSpan.getEnd());
assertEquals(true, personName.isClearAdaptiveDataSet());
-
+
assertEquals(0, sampleStream.read().getNames().length);
-
+
assertNull(sampleStream.read());
}
-
+
@Test
public void testParsingDutchSample() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.NL, "conll2002-nl.sample");
-
+
NameSample personName = sampleStream.read();
-
+
assertEquals(0, personName.getNames().length);
assertTrue(personName.isClearAdaptiveDataSet());
-
+
personName = sampleStream.read();
-
+
assertFalse(personName.isClearAdaptiveDataSet());
-
+
assertNull(sampleStream.read());
}
-
+
@Test
public void testReset() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.NL, "conll2002-nl.sample");
-
+
NameSample sample = sampleStream.read();
-
+
sampleStream.reset();
-
+
assertEquals(sample, sampleStream.read());
}
}
Modified: opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamTest.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamTest.java?rev=1591889&r1=1591888&r2=1591889&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamTest.java (original)
+++ opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/formats/Conll03NameSampleStreamTest.java Fri May 2 12:34:23 2014
@@ -38,10 +38,10 @@ public class Conll03NameSampleStreamTest
private static final String ENGLISH_SAMPLE = "conll2003-en.sample";
private static final String GERMAN_SAMPLE = "conll2003-de.sample";
-
-
+
+
private static ObjectStream<NameSample> openData(LANGUAGE lang, String name) throws IOException {
- InputStreamFactory in = new ResourceAsStreamFactory(Conll03NameSampleStreamTest.class,
+ InputStreamFactory in = new ResourceAsStreamFactory(Conll03NameSampleStreamTest.class,
"/opennlp/tools/formats/" + name);
return new Conll03NameSampleStream(lang, in, Conll02NameSampleStream.GENERATE_PERSON_ENTITIES);
@@ -73,40 +73,40 @@ public class Conll03NameSampleStreamTest
assertNull(sampleStream.read());
}
-
+
@Test(expected=IOException.class)
public void testParsingEnglishSampleWithGermanAsLanguage() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.DE, ENGLISH_SAMPLE);
sampleStream.read();
}
-
+
@Test(expected=IOException.class)
public void testParsingGermanSampleWithEnglishAsLanguage() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.EN, GERMAN_SAMPLE);
sampleStream.read();
}
-
+
@Test
public void testParsingGermanSample() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.DE, GERMAN_SAMPLE);
-
+
NameSample personName = sampleStream.read();
assertNotNull(personName);
-
+
assertEquals(5, personName.getSentence().length);
assertEquals(0, personName.getNames().length);
assertEquals(true, personName.isClearAdaptiveDataSet());
}
-
+
@Test
public void testReset() throws IOException {
ObjectStream<NameSample> sampleStream = openData(LANGUAGE.DE, GERMAN_SAMPLE);
-
+
NameSample sample = sampleStream.read();
-
+
sampleStream.reset();
-
+
assertEquals(sample, sampleStream.read());
}
}
\ No newline at end of file