You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2014/02/06 16:37:24 UTC
svn commit: r1565309 - in
/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools: formats/
formats/ad/ namefind/ parser/chunking/ parser/treeinsert/ util/
Author: joern
Date: Thu Feb 6 15:37:23 2014
New Revision: 1565309
URL: http://svn.apache.org/r1565309
Log:
OPENNLP-600 Reverted a couple of changes. The stream classes need to get a new constructor. Ohter classes should for now just continue using the deprecated API. BeamSearch was not changed, only commented code was added.
Removed:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/MockInputStreamFactory.java
Modified:
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java Thu Feb 6 15:37:23 2014
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import opennlp.tools.namefind.NameSample;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -55,9 +54,9 @@ public class BioNLP2004NameSampleStream
public BioNLP2004NameSampleStream(InputStream in, int types) {
try {
- this.lineStream = new PlainTextByLineStream(new MockInputStreamFactory(in), "UTF-8");
+ this.lineStream = new PlainTextByLineStream(in, "UTF-8");
System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (IOException e) {
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java Thu Feb 6 15:37:23 2014
@@ -26,7 +26,6 @@ import java.util.List;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.InvalidFormatException;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -81,9 +80,9 @@ public class Conll02NameSampleStream imp
this.lang = lang;
try {
- this.lineStream = new PlainTextByLineStream(new MockInputStreamFactory(in), "UTF-8");
+ this.lineStream = new PlainTextByLineStream(in, "UTF-8");
System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (IOException e) {
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java Thu Feb 6 15:37:23 2014
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import opennlp.tools.namefind.NameSample;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -68,9 +67,9 @@ public class Conll03NameSampleStream imp
this.lang = lang;
try {
- this.lineStream = new PlainTextByLineStream(new MockInputStreamFactory(in), "UTF-8");
+ this.lineStream = new PlainTextByLineStream(in, "UTF-8");
System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (IOException e) {
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java Thu Feb 6 15:37:23 2014
@@ -26,7 +26,6 @@ import java.util.List;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.InvalidFormatException;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -88,9 +87,9 @@ public class EvalitaNameSampleStream imp
this.lang = lang;
try {
- this.lineStream = new PlainTextByLineStream(new MockInputStreamFactory(in), "UTF-8");
+ this.lineStream = new PlainTextByLineStream(in, "UTF-8");
System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (IOException e) {
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java Thu Feb 6 15:37:23 2014
@@ -24,7 +24,6 @@ import java.io.PrintStream;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.tokenize.SimpleTokenizer;
import opennlp.tools.util.FilterObjectStream;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.PlainTextByLineStream;
/**
@@ -53,7 +52,7 @@ public class LeipzigDoccatSampleStream e
*/
LeipzigDoccatSampleStream(String language, int sentencesPerDocument,
InputStream in) throws IOException {
- super(new PlainTextByLineStream(new MockInputStreamFactory(in), "UTF-8"));
+ super(new PlainTextByLineStream(in, "UTF-8"));
System.setOut(new PrintStream(System.out, true, "UTF-8"));
this.language = language;
this.sentencesPerDocument = sentencesPerDocument;
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java Thu Feb 6 15:37:23 2014
@@ -12,15 +12,13 @@
* limitations under the License.
* under the License.
*/
+
package opennlp.tools.formats;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Locale;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -29,10 +27,10 @@ import opennlp.tools.util.StringUtil;
/**
* This class helps to read the US Census data from the files to build a
- * StringList for each dictionary entry in the name-finder dictionary. The
- * entries in the source file are as follows:
+ * StringList for each dictionary entry in the name-finder dictionary.
+ * The entries in the source file are as follows:
* <p>
- * SMITH 1.006 1.006 1
+ * SMITH 1.006 1.006 1
* <p>
* <ul>
* <li>The first field is the name (in ALL CAPS).
@@ -47,14 +45,14 @@ public class NameFinderCensus90NameStrea
private final Locale locale;
private final Charset encoding;
- private ObjectStream<String> lineStream;
+ private final ObjectStream<String> lineStream;
/**
* This constructor takes an ObjectStream and initializes the class to handle
* the stream.
*
- * @param lineStream an <code>ObjectSteam<String></code> that represents the
- * input file to be attached to this class.
+ * @param lineStream an <code>ObjectSteam<String></code> that represents the
+ * input file to be attached to this class.
*/
public NameFinderCensus90NameStream(ObjectStream<String> lineStream) {
this.locale = new Locale("en"); // locale is English
@@ -64,32 +62,24 @@ public class NameFinderCensus90NameStrea
}
/**
- * This constructor takes an
- * <code>InputStream</code> and a
- * <code>Charset</code> and opens an associated stream object with the
- * specified encoding specified.
+ * This constructor takes an <code>InputStream</code> and a <code>Charset</code>
+ * and opens an associated stream object with the specified encoding specified.
*
- * @param in an <code>InputStream</code> for the input file.
- * @param encoding the <code>Charset</code> to apply to the input stream.
+ * @param in an <code>InputStream</code> for the input file.
+ * @param encoding the <code>Charset</code> to apply to the input stream.
*/
public NameFinderCensus90NameStream(InputStream in, Charset encoding) {
this.locale = new Locale("en"); // locale is English
this.encoding = encoding;
-
- try {
- this.lineStream = new PlainTextByLineStream(new MockInputStreamFactory(in), this.encoding);
- } catch (IOException ex) {
-
- throw new RuntimeException(ex);
- }
+ this.lineStream = new PlainTextByLineStream(in, this.encoding);
}
public StringList read() throws IOException {
String line = lineStream.read();
StringList name = null;
- if ((line != null)
- && (!StringUtil.isEmpty(line))) {
+ if ((line != null) &&
+ (!StringUtil.isEmpty(line))) {
String name2;
// find the location of the name separator in the line of data.
int pos = line.indexOf(' ');
@@ -97,15 +87,15 @@ public class NameFinderCensus90NameStrea
String parsed = line.substring(0, pos);
// the data is in ALL CAPS ... so the easiest way is to convert
// back to standard mixed case.
- if ((parsed.length() > 2)
- && (parsed.startsWith("MC"))) {
- name2 = parsed.substring(0, 1).toUpperCase(locale)
- + parsed.substring(1, 2).toLowerCase(locale)
- + parsed.substring(2, 3).toUpperCase(locale)
- + parsed.substring(3).toLowerCase(locale);
+ if ((parsed.length() > 2) &&
+ (parsed.startsWith("MC"))) {
+ name2 = parsed.substring(0,1).toUpperCase(locale) +
+ parsed.substring(1,2).toLowerCase(locale) +
+ parsed.substring(2,3).toUpperCase(locale) +
+ parsed.substring(3).toLowerCase(locale);
} else {
- name2 = parsed.substring(0, 1).toUpperCase(locale)
- + parsed.substring(1).toLowerCase(locale);
+ name2 = parsed.substring(0,1).toUpperCase(locale) +
+ parsed.substring(1).toLowerCase(locale);
}
name = new StringList(new String[]{name2});
}
@@ -121,4 +111,5 @@ public class NameFinderCensus90NameStrea
public void close() throws IOException {
lineStream.close();
}
+
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java Thu Feb 6 15:37:23 2014
@@ -29,7 +29,6 @@ import opennlp.tools.formats.ad.ADSenten
import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node;
import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement;
import opennlp.tools.namefind.NameSample;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.StringUtil;
@@ -92,8 +91,8 @@ public class ADChunkSampleStream impleme
try {
this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- new MockInputStreamFactory(in), charsetName));
- } catch (IOException e) {
+ in, charsetName));
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java Thu Feb 6 15:37:23 2014
@@ -35,7 +35,6 @@ import opennlp.tools.formats.ad.ADSenten
import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node;
import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement;
import opennlp.tools.namefind.NameSample;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -192,9 +191,9 @@ public class ADNameSampleStream implemen
try {
this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- new MockInputStreamFactory(in), charsetName));
+ in, charsetName));
this.splitHyphenatedTokens = splitHyphenatedTokens;
- } catch (IOException e) {
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java Thu Feb 6 15:37:23 2014
@@ -29,7 +29,6 @@ import opennlp.tools.formats.ad.ADSenten
import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node;
import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement;
import opennlp.tools.postag.POSSample;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -82,10 +81,10 @@ public class ADPOSSampleStream implement
try {
this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- new MockInputStreamFactory(in), charsetName));
+ in, charsetName));
this.expandME = expandME;
this.isIncludeFeatures = includeFeatures;
- } catch (IOException e) {
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java Thu Feb 6 15:37:23 2014
@@ -29,7 +29,6 @@ import java.util.regex.Pattern;
import opennlp.tools.formats.ad.ADSentenceStream.Sentence;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.sentdetect.lang.Factory;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -82,8 +81,8 @@ public class ADSentenceSampleStream impl
boolean includeHeadlines) {
try {
this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- new MockInputStreamFactory(in), charsetName));
- } catch (IOException e) {
+ in, charsetName));
+ } catch (UnsupportedEncodingException e) {
// UTF-8 is available on all JVMs, will never happen
throw new IllegalStateException(e);
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java Thu Feb 6 15:37:23 2014
@@ -23,10 +23,7 @@ import java.util.List;
import java.util.Map;
import opennlp.tools.ml.model.Event;
-import opennlp.tools.ml.model.EventStream;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.featuregen.AdditionalContextFeatureGenerator;
import opennlp.tools.util.featuregen.WindowFeatureGenerator;
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java Thu Feb 6 15:37:23 2014
@@ -29,7 +29,6 @@ import opennlp.tools.parser.Parse;
import opennlp.tools.parser.ParseSampleStream;
import opennlp.tools.parser.ParserEventTypeEnum;
import opennlp.tools.util.InvalidFormatException;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
@@ -205,7 +204,7 @@ public class ParserEventStream extends A
if (fun) {
Parse.useFunctionTags(true);
}
- opennlp.tools.ml.model.EventStream es = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new MockInputStreamFactory(System.in),"UTF-8")), rules, etype, dict);
+ opennlp.tools.ml.model.EventStream es = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.InputStreamReader(System.in))), rules, etype, dict);
while (es.hasNext()) {
System.out.println(es.next());
}
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java Thu Feb 6 15:37:23 2014
@@ -36,7 +36,6 @@ import opennlp.tools.parser.Parse;
import opennlp.tools.parser.ParseSampleStream;
import opennlp.tools.parser.ParserEventTypeEnum;
import opennlp.tools.util.InvalidFormatException;
-import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -380,7 +379,7 @@ public class ParserEventStream extends A
if (fun) {
Parse.useFunctionTags(true);
}
- opennlp.tools.ml.model.EventStream es = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new MockInputStreamFactory(System.in),"UTF-8")), rules, etype, dict);
+ opennlp.tools.ml.model.EventStream es = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.InputStreamReader(System.in))), rules, etype, dict);
while (es.hasNext()) {
Event e = es.next();
if (model != null) {
Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java?rev=1565309&r1=1565308&r2=1565309&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java Thu Feb 6 15:37:23 2014
@@ -14,16 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package opennlp.tools.util;
import java.util.Arrays;
import java.util.List;
import opennlp.tools.ml.model.MaxentModel;
-import opennlp.tools.ml.model.RealValueFileEventStream;
/**
- * Performs k-best search over sequence. This is based on the description in
+ * Performs k-best search over sequence. This is based on the description in
* Ratnaparkhi (1998), PhD diss, Univ. of Pennsylvania.
*
* @see Sequence
@@ -33,10 +33,12 @@ import opennlp.tools.ml.model.RealValueF
public class BeamSearch<T> {
private static final Object[] EMPTY_ADDITIONAL_CONTEXT = new Object[0];
+
protected int size;
protected BeamSearchContextGenerator<T> cg;
protected MaxentModel model;
private SequenceValidator<T> validator;
+
private double[] probs;
private Cache contextsCache;
private static final int zeroLog = -100000;
@@ -44,22 +46,21 @@ public class BeamSearch<T> {
/**
* Creates new search object.
*
- * @param size The size of the beam (k).
- * @param cg the context generator for the model.
- * @param model the model for assigning probabilities to the sequence
- * outcomes.
+ * @param size The size of the beam (k).
+ * @param cg the context generator for the model.
+ * @param model the model for assigning probabilities to the sequence outcomes.
*/
public BeamSearch(int size, BeamSearchContextGenerator<T> cg, MaxentModel model) {
this(size, cg, model, null, 0);
}
public BeamSearch(int size, BeamSearchContextGenerator<T> cg, MaxentModel model,
- int cacheSize) {
- this(size, cg, model, null, cacheSize);
+ int cacheSize) {
+ this (size, cg, model, null, cacheSize);
}
public BeamSearch(int size, BeamSearchContextGenerator<T> cg, MaxentModel model,
- SequenceValidator<T> validator, int cacheSize) {
+ SequenceValidator<T> validator, int cacheSize) {
this.size = size;
this.cg = cg;
@@ -74,7 +75,8 @@ public class BeamSearch<T> {
}
/**
- * Note: This method will be private in the future because clients can now
+ * Note:
+ * This method will be private in the future because clients can now
* pass a validator to validate the sequence.
*
* @see SequenceValidator
@@ -83,7 +85,8 @@ public class BeamSearch<T> {
if (validator != null) {
return validator.validSequence(i, inputSequence, outcomesSequence, outcome);
- } else {
+ }
+ else {
return true;
}
}
@@ -95,12 +98,10 @@ public class BeamSearch<T> {
/**
* Returns the best sequence of outcomes based on model for this object.
*
- * @param numSequences The maximum number of sequences to be returned.
- * @param sequence The input sequence.
- * @param additionalContext An Object[] of additional context. This is passed
- * to the context generator blindly with the
- * assumption that the context are appropiate.
- * @param minSequenceScore A lower bound on the score of a returned sequence.
+ * @param numSequences The maximum number of sequences to be returned.
+ * @param sequence The input sequence.
+ * @param additionalContext An Object[] of additional context. This is passed to the context generator blindly with the assumption that the context are appropiate.
+ * @param minSequenceScore A lower bound on the score of a returned sequence.
* @return An array of the top ranked sequences of outcomes.
*/
public Sequence[] bestSequences(int numSequences, T[] sequence, Object[] additionalContext, double minSequenceScore) {
@@ -123,23 +124,15 @@ public class BeamSearch<T> {
String[] outcomes = tmpOutcomes.toArray(new String[tmpOutcomes.size()]);
String[] contexts = cg.getContext(i, sequence, outcomes, additionalContext);
double[] scores;
- // float[] realValues = RealValueFileEventStream.parseContexts(contexts);
if (contextsCache != null) {
scores = (double[]) contextsCache.get(contexts);
if (scores == null) {
- // if (realValues != null) {
- // scores = model.eval(contexts, realValues);
- // } else {
- scores = model.eval(contexts, probs);
- // }
- contextsCache.put(contexts, scores);
- }
- } else {
- // if (realValues != null) {
- // scores = model.eval(contexts, realValues);
- // } else {
scores = model.eval(contexts, probs);
- //}
+ contextsCache.put(contexts,scores);
+ }
+ }
+ else {
+ scores = model.eval(contexts, probs);
}
double[] temp_scores = new double[scores.length];
@@ -149,12 +142,11 @@ public class BeamSearch<T> {
Arrays.sort(temp_scores);
- double min = temp_scores[Math.max(0, scores.length - size)];
+ double min = temp_scores[Math.max(0,scores.length-size)];
for (int p = 0; p < scores.length; p++) {
- if (scores[p] < min) {
+ if (scores[p] < min)
continue; //only advance first "size" outcomes
- }
String out = model.getOutcome(p);
if (validSequence(i, sequence, outcomes, out)) {
Sequence ns = new Sequence(top, out, scores[p]);
@@ -197,21 +189,17 @@ public class BeamSearch<T> {
/**
* Returns the best sequence of outcomes based on model for this object.
*
- * @param sequence The input sequence.
- * @param additionalContext An Object[] of additional context. This is passed
- * to the context generator blindly with the
- * assumption that the context are appropiate.
+ * @param sequence The input sequence.
+ * @param additionalContext An Object[] of additional context. This is passed to the context generator blindly with the assumption that the context are appropiate.
*
- * @return The top ranked sequence of outcomes or null if no sequence could be
- * found
+ * @return The top ranked sequence of outcomes or null if no sequence could be found
*/
public Sequence bestSequence(T[] sequence, Object[] additionalContext) {
- Sequence sequences[] = bestSequences(1, sequence, additionalContext, zeroLog);
-
- if (sequences.length > 0) {
+ Sequence sequences[] = bestSequences(1, sequence, additionalContext,zeroLog);
+
+ if (sequences.length > 0)
return sequences[0];
- } else {
+ else
return null;
- }
}
}