You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jb...@apache.org on 2011/04/14 05:38:36 UTC
svn commit: r1091994 - in
/incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp:
maxent/DoubleStringPair.java maxent/ModelApplier.java
maxent/ModelTrainer.java perceptron/PerceptronModel.java
perceptron/PerceptronTrainer.java
Author: jbaldrid
Date: Thu Apr 14 03:38:36 2011
New Revision: 1091994
URL: http://svn.apache.org/viewvc?rev=1091994&view=rev
Log:
Fixes to Perceptron (improved normalization, improved checking of training accuracy); improved command line trainer so that a Perceptron model is actually stored, so that options are passed to Perceptron training, and so that output of ModelApplier is one line per test event.
Added:
incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/DoubleStringPair.java
Modified:
incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelApplier.java
incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelTrainer.java
incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronModel.java
incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java
Added: incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/DoubleStringPair.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/DoubleStringPair.java?rev=1091994&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/DoubleStringPair.java (added)
+++ incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/DoubleStringPair.java Thu Apr 14 03:38:36 2011
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package opennlp.maxent;
+
+public class DoubleStringPair implements Comparable<DoubleStringPair> {
+
+ final public String stringValue;
+ final public double doubleValue;
+
+ public DoubleStringPair (double d, String s) {
+ doubleValue = d;
+ stringValue = s;
+ }
+
+ public int compareTo(DoubleStringPair p) {
+ return Double.compare(doubleValue,p.doubleValue);
+ }
+
+}
Modified: incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelApplier.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelApplier.java?rev=1091994&r1=1091993&r2=1091994&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelApplier.java (original)
+++ incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelApplier.java Thu Apr 14 03:38:36 2011
@@ -50,7 +50,7 @@ public class ModelApplier {
private void eval(Event event, boolean real) {
- String outcome = event.getOutcome();
+ String outcome = event.getOutcome(); // Is ignored
String[] context = event.getContext();
double[] ocs;
@@ -61,20 +61,17 @@ public class ModelApplier {
ocs = _model.eval(context, values);
}
- int best = 0;
- for (int i = 1; i < ocs.length; i++)
- if (ocs[i] > ocs[best])
- best = i;
-
- String predictedLabel = _model.getOutcome(best);
- String madeError = "+";
- if (predictedLabel.equals(outcome))
- madeError = "";
-
- System.out.println(counter + "\t0:" + outcome + "\t0:"
- + _model.getOutcome(best) + "\t" + madeError + "\t"
- + ROUNDED_FORMAT.format(ocs[best]));
- counter++;
+ int numOutcomes = ocs.length;
+ DoubleStringPair[] result = new DoubleStringPair[numOutcomes];
+ for (int i=0; i<numOutcomes; i++)
+ result[i] = new DoubleStringPair(ocs[i], _model.getOutcome(i));
+
+ java.util.Arrays.sort(result);
+
+ // Print the most likely outcome first, down to the least likely.
+ for (int i=numOutcomes-1; i>=0; i--)
+ System.out.print(result[i].stringValue + " " + result[i].doubleValue + " ");
+ System.out.println();
}
@@ -89,10 +86,16 @@ public class ModelApplier {
* java ModelApplier modelFile dataFile
*/
public static void main(String[] args) {
+
String dataFileName, modelFileName;
boolean real = false;
String type = "maxent";
int ai = 0;
+
+ if (args.length == 0) {
+ usage();
+ }
+
if (args.length > 0) {
while (args[ai].startsWith("-")) {
if (args[ai].equals("-real")) {
@@ -104,28 +107,25 @@ public class ModelApplier {
}
ai++;
}
+
modelFileName = args[ai++];
dataFileName = args[ai++];
ModelApplier predictor = null;
try {
- MaxentModel m = new GenericModelReader(new File(modelFileName))
- .getModel();
+ MaxentModel m = new GenericModelReader(new File(modelFileName)).getModel();
predictor = new ModelApplier(m);
} catch (Exception e) {
e.printStackTrace();
System.exit(0);
}
- System.out.println("=== Predictions on test data ===\n");
- System.out.println(" inst# actual predicted error prediction");
try {
EventStream es = new BasicEventStream(new PlainTextByLineDataStream(
new FileReader(new File(dataFileName))), ",");
- while (es.hasNext()) {
+ while (es.hasNext())
predictor.eval(es.next(), real);
- }
return;
} catch (Exception e) {
Modified: incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelTrainer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelTrainer.java?rev=1091994&r1=1091993&r2=1091994&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelTrainer.java (original)
+++ incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/maxent/ModelTrainer.java Thu Apr 14 03:38:36 2011
@@ -25,10 +25,12 @@ import java.io.FileReader;
import opennlp.maxent.io.GISModelWriter;
import opennlp.maxent.io.SuffixSensitiveGISModelWriter;
import opennlp.model.AbstractModel;
+import opennlp.model.AbstractModelWriter;
import opennlp.model.EventStream;
import opennlp.model.OnePassDataIndexer;
import opennlp.model.OnePassRealValueDataIndexer;
import opennlp.perceptron.PerceptronTrainer;
+import opennlp.perceptron.SuffixSensitivePerceptronModelWriter;
/**
* Main class which calls the GIS procedure after building the EventStream from
@@ -93,29 +95,38 @@ public class ModelTrainer {
} else {
es = new RealBasicEventStream(new PlainTextByLineDataStream(datafr));
}
- GIS.SMOOTHING_OBSERVATION = SMOOTHING_OBSERVATION;
+
+ File outputFile = new File(modelFileName);
+
+ AbstractModelWriter writer;
+
AbstractModel model;
if (type.equals("maxent")) {
+ GIS.SMOOTHING_OBSERVATION = SMOOTHING_OBSERVATION;
if (!real) {
model = GIS.trainModel(es, maxit, cutoff, sigma);
} else {
- model = GIS.trainModel(maxit, new OnePassRealValueDataIndexer(es, 0),
- USE_SMOOTHING);
+ model = GIS.trainModel(maxit,
+ new OnePassRealValueDataIndexer(es, cutoff),
+ USE_SMOOTHING);
}
+
+ writer = new SuffixSensitiveGISModelWriter(model, outputFile);
+
} else if (type.equals("perceptron")) {
- System.err.println("Perceptron training");
- model = new PerceptronTrainer().trainModel(10, new OnePassDataIndexer(
- es, 0), 0);
+ //System.err.println("Perceptron training");
+ model = new PerceptronTrainer().trainModel(maxit, new OnePassDataIndexer(es, cutoff), cutoff);
+
+ writer = new SuffixSensitivePerceptronModelWriter(model, outputFile);
+
} else {
- System.err.println("Unknown model type: " + type);
- model = null;
+ throw new RuntimeException("Unknown model type: " + type);
}
- File outputFile = new File(modelFileName);
- GISModelWriter writer = new SuffixSensitiveGISModelWriter(model,
- outputFile);
writer.persist();
+
+
} catch (Exception e) {
System.out.print("Unable to create model due to exception: ");
System.out.println(e);
Modified: incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronModel.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronModel.java?rev=1091994&r1=1091993&r2=1091994&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronModel.java (original)
+++ incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronModel.java Thu Apr 14 03:38:36 2011
@@ -87,29 +87,16 @@ public class PerceptronModel extends Abs
}
}
if (normalize) {
+ int numOutcomes = model.getNumOutcomes();
+ for (int oid = 0; oid < numOutcomes; oid++)
+ prior[oid] = Math.exp(prior[oid]);
+
double normal = 0.0;
- double min = prior[0];
- for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
- if (prior[oid] < min) {
- min = prior[oid];
- }
- }
- for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
- if (min < 0) {
- prior[oid]+=(-1*min);
- }
- normal += prior[oid];
- }
- if (normal == 0.0) {
- for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
- prior[oid] = (double) 1/model.getNumOutcomes();
- }
- }
- else {
- for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
- prior[oid] /= normal;
- }
- }
+ for (int oid = 0; oid < numOutcomes; oid++)
+ normal += prior[oid];
+
+ for (int oid = 0; oid < numOutcomes; oid++)
+ prior[oid] /= normal;
}
return prior;
}
Modified: incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java?rev=1091994&r1=1091993&r2=1091994&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java (original)
+++ incubator/opennlp/trunk/opennlp-maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java Thu Apr 14 03:38:36 2011
@@ -128,10 +128,13 @@ public class PerceptronTrainer {
for (int pi = 0; pi < numPreds; pi++) {
params[pi] = new MutableContext(allOutcomesPattern,new double[numOutcomes]);
- if (useAverage) averageParams[pi] = new MutableContext(allOutcomesPattern,new double[numOutcomes]);
+ if (useAverage)
+ averageParams[pi] =
+ new MutableContext(allOutcomesPattern,new double[numOutcomes]);
for (int aoi=0;aoi<numOutcomes;aoi++) {
params[pi].setParameter(aoi, 0.0);
- if (useAverage) averageParams[pi].setParameter(aoi, 0.0);
+ if (useAverage)
+ averageParams[pi].setParameter(aoi, 0.0);
}
}
modelDistribution = new double[numOutcomes];
@@ -141,12 +144,10 @@ public class PerceptronTrainer {
display("...done.\n");
/*************** Create and return the model ******************/
- if (useAverage) {
+ if (useAverage)
return new PerceptronModel(averageParams, predLabels, outcomeLabels);
- }
- else {
+ else
return new PerceptronModel(params, predLabels, outcomeLabels);
- }
}
private void display(String s) {
@@ -155,7 +156,11 @@ public class PerceptronTrainer {
}
private void findParameters(int iterations) {
+
display("Performing " + iterations + " iterations.\n");
+
+ int numTimesSameAccuracy = 0;
+ double prevAccuracy = 0.0;
for (int i = 1; i <= iterations; i++) {
if (i < 10)
display(" " + i + ": ");
@@ -164,140 +169,128 @@ public class PerceptronTrainer {
else
display(i + ": ");
nextIteration(i);
+
+ // Need to do this for the full set to get a representative
+ // accuracy -- doing it while training is biased because the
+ // events are ordered according to their outcomes.
+ double currAccuracy = trainingStats(averageParams);
+
+ if (currAccuracy == prevAccuracy) {
+ numTimesSameAccuracy++;
+ } else {
+ prevAccuracy = currAccuracy;
+ numTimesSameAccuracy = 0;
+ }
+
+ // If the accuracy hasn't changed for four iterations, stop training.
+ if (numTimesSameAccuracy == 4) {
+ display("Accuracy repeated 4 times, stopping training.\n");
+ break;
+ }
}
- if (useAverage) {
+ if (useAverage)
trainingStats(averageParams);
- }
- else {
+ else
trainingStats(params);
- }
+
// kill a bunch of these big objects now that we don't need them
numTimesEventsSeen = null;
contexts = null;
}
- /* Compute one iteration of Perceptron and return log-likelihood.*/
+ /* Compute one iteration of Perceptron.*/
private void nextIteration(int iteration) {
iteration--; //move to 0-based index
- int numCorrect = 0;
int oei = 0;
- for (int ei = 0; ei < numUniqueEvents; ei++,oei++) {
- //Arrays.sort(contexts[ei]); only needed for debugging
+ for (int ei = 0; ei < numUniqueEvents; ei++, oei++) {
for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
- //System.err.print("contexts["+ei+"]=");for (int ci=0;ci<contexts[ei].length;ci++) { System.err.print(" "+contexts[ei][ci]+" ");} System.err.println();
- for (int oi = 0; oi < numOutcomes; oi++) {
+
+ for (int oi = 0; oi < numOutcomes; oi++)
modelDistribution[oi] = 0;
- }
- if (values != null) {
+
+ if (values != null)
PerceptronModel.eval(contexts[ei], values[ei], modelDistribution, evalParams,false);
- }
- else {
+ else
PerceptronModel.eval(contexts[ei], null, modelDistribution, evalParams, false);
- }
+
int max = 0;
- for (int oi = 1; oi < numOutcomes; oi++) {
- if (modelDistribution[oi] > modelDistribution[max]) {
+ for (int oi = 1; oi < numOutcomes; oi++)
+ if (modelDistribution[oi] > modelDistribution[max])
max = oi;
- }
- }
- boolean correct = max == outcomeList[oei];
- if (correct) {
- numCorrect ++;
- }
+
for (int oi = 0;oi<numOutcomes;oi++) {
- if (oi == outcomeList[oei]) {
- if (modelDistribution[oi] <= 0) {
- for (int ci = 0; ci < contexts[ei].length; ci++) {
- int pi = contexts[ei][ci];
- if (values == null) {
- params[pi].updateParameter(oi, 1);
- }
- else {
- params[pi].updateParameter(oi, values[ei][ci]);
- }
- if (useAverage) {
- if (updates[pi][oi][VALUE] != 0) {
- averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
- //System.err.println("p avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
- }
- //System.err.println("p updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
- updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
- updates[pi][oi][ITER] = iteration;
- updates[pi][oi][EVENT] = ei;
- }
- }
- }
- }
- else {
- if (modelDistribution[oi] > 0) {
- for (int ci = 0; ci < contexts[ei].length; ci++) {
- int pi = contexts[ei][ci];
- if (values == null) {
- params[pi].updateParameter(oi, -1);
- }
- else {
- params[pi].updateParameter(oi, -1*values[ei][ci]);
- }
- if (useAverage) {
- if (updates[pi][oi][VALUE] != 0) {
- averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
- //System.err.println("d avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
- }
- //System.err.println(ei+" d updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
- updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
- updates[pi][oi][ITER] = iteration;
- updates[pi][oi][EVENT] = ei;
- }
- }
- }
- }
- }
+ int updateValue = -1;
+ if (oi == outcomeList[oei])
+ updateValue = 1;
+
+ if (modelDistribution[oi]*updateValue <= 0) {
+ for (int ci = 0; ci < contexts[ei].length; ci++) {
+ int pi = contexts[ei][ci];
+ if (values == null)
+ params[pi].updateParameter(oi, updateValue);
+ else
+ params[pi].updateParameter(oi, updateValue*values[ei][ci]);
+
+ if (useAverage) {
+
+ if (updates[pi][oi][VALUE] != 0)
+ averageParams[pi].updateParameter(oi,
+ updates[pi][oi][VALUE] *
+ (numEvents * (iteration-updates[pi][oi][ITER])
+ + (ei-updates[pi][oi][EVENT])));
+
+ updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
+ updates[pi][oi][ITER] = iteration;
+ updates[pi][oi][EVENT] = ei;
+ }
+ }
+ }
+ }
}
}
+
//finish average computation
double totIterations = (double) iterations*numEvents;
if (useAverage && iteration == iterations-1) {
for (int pi = 0; pi < numPreds; pi++) {
double[] predParams = averageParams[pi].getParameters();
for (int oi = 0;oi<numOutcomes;oi++) {
- if (updates[pi][oi][VALUE] != 0) {
- predParams[oi] += updates[pi][oi][VALUE]*(numEvents*(iterations-updates[pi][oi][ITER])-updates[pi][oi][EVENT]);
- }
+ if (updates[pi][oi][VALUE] != 0)
+ predParams[oi] +=
+ updates[pi][oi][VALUE] *
+ (numEvents * (iterations-updates[pi][oi][ITER])
+ - updates[pi][oi][EVENT]);
+
if (predParams[oi] != 0) {
predParams[oi] /=totIterations;
averageParams[pi].setParameter(oi, predParams[oi]);
- //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iterations+","+0+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
}
}
}
}
- display(". ("+numCorrect+"/"+numEvents+") "+((double) numCorrect / numEvents) + "\n");
}
- private void trainingStats(MutableContext[] params) {
+ private double trainingStats(MutableContext[] params) {
int numCorrect = 0;
- for (int ei = 0; ei < numUniqueEvents; ei++) {
+ int oei = 0;
+ for (int ei = 0; ei < numUniqueEvents; ei++, oei++) {
for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
- for (int oi = 0; oi < numOutcomes; oi++) {
+ for (int oi = 0; oi < numOutcomes; oi++)
modelDistribution[oi] = 0;
- }
- if (values != null) {
+ if (values != null)
PerceptronModel.eval(contexts[ei], values[ei], modelDistribution, evalParams,false);
- }
- else {
+ else
PerceptronModel.eval(contexts[ei], null, modelDistribution, evalParams, false);
- }
int max = 0;
- for (int oi = 1; oi < numOutcomes; oi++) {
- if (modelDistribution[oi] > modelDistribution[max]) {
+ for (int oi = 1; oi < numOutcomes; oi++)
+ if (modelDistribution[oi] > modelDistribution[max])
max = oi;
- }
- }
- if (max == outcomeList[ei]) {
+ if (max == outcomeList[oei])
numCorrect ++;
- }
}
}
- display(". ("+numCorrect+"/"+numEvents+") "+((double) numCorrect / numEvents) + "\n");
+ double trainingAccuracy = (double) numCorrect / numEvents;
+ display(". ("+numCorrect+"/"+numEvents+") "+ trainingAccuracy + "\n");
+ return trainingAccuracy;
}
}