You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/04/29 06:47:03 UTC
[01/10] incubator-joshua git commit: Mostly a refactor for improved
readability
Repository: incubator-joshua
Updated Branches:
refs/heads/master 051cc489d -> 3b2a17c0c
Mostly a refactor for improved readability
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/282d5267
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/282d5267
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/282d5267
Branch: refs/heads/master
Commit: 282d5267ee04c60efd810b3352f24e6c99786ea6
Parents: cc55600
Author: Felix Hieber <fh...@amazon.com>
Authored: Wed Feb 3 15:17:30 2016 +0100
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Thu Apr 28 15:42:05 2016 -0700
----------------------------------------------------------------------
src/joshua/decoder/chart_parser/Cell.java | 62 ++++++++------------
src/joshua/decoder/hypergraph/ForestWalker.java | 2 +-
src/joshua/decoder/hypergraph/HGNode.java | 2 -
3 files changed, 25 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/282d5267/src/joshua/decoder/chart_parser/Cell.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/chart_parser/Cell.java b/src/joshua/decoder/chart_parser/Cell.java
index 5ab2a15..d8d16d8 100644
--- a/src/joshua/decoder/chart_parser/Cell.java
+++ b/src/joshua/decoder/chart_parser/Cell.java
@@ -18,13 +18,16 @@
*/
package joshua.decoder.chart_parser;
-import java.util.ArrayList;
-import java.util.Arrays;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.Map.Entry;
import java.util.logging.Logger;
import joshua.decoder.ff.FeatureFunction;
@@ -254,52 +257,35 @@ class Cell {
*/
private void ensureSorted() {
if (null == this.sortedNodes) {
- // Get sortedNodes.
- HGNode[] nodesArray = new HGNode[this.nodesSigTbl.size()];
- int i = 0;
- for (HGNode node : this.nodesSigTbl.values())
- nodesArray[i++] = node;
-
- /**
- * sort the node in an decreasing-LogP order
- * */
- Arrays.sort(nodesArray, HGNode.inverseLogPComparator);
-
- this.sortedNodes = new ArrayList<HGNode>();
- for (HGNode node : nodesArray) {
+
+ // get sortedNodes.
+ this.sortedNodes = new ArrayList<>(this.nodesSigTbl.size());
+ for (HGNode node : this.nodesSigTbl.values()) {
this.sortedNodes.add(node);
}
- // TODO: we cannot create new SuperItem here because the DotItem link to them
+ // sort the node in an decreasing-LogP order
+ this.sortedNodes.sort(HGNode.inverseLogPComparator);
- // Update superNodesTbl
- List<SuperNode> tem_list = new ArrayList<SuperNode>(this.superNodesTbl.values());
- for (SuperNode t_si : tem_list) {
- t_si.nodes.clear();
+ // TODO: we cannot create new SuperItem here because the DotItem link to them.
+ // Thus, we clear nodes from existing SuperNodes
+ for (SuperNode superNode : this.superNodesTbl.values()) {
+ superNode.nodes.clear();
}
- for (HGNode it : this.sortedNodes) {
- SuperNode si = this.superNodesTbl.get(it.lhs);
- if (null == si) { // sanity check
- throw new RuntimeException("Does not have super Item, have to exist");
- }
- si.nodes.add(it);
+ for (HGNode node : this.sortedNodes) {
+ SuperNode superNode = this.superNodesTbl.get(node.lhs);
+ checkNotNull(superNode, "Does not have super Item, have to exist");
+ superNode.nodes.add(node);
}
- // Remove SuperNodes who may not contain any node any more due to pruning
- List<Integer> toRemove = new ArrayList<Integer>();
- for (Integer k : this.superNodesTbl.keySet()) {
- if (this.superNodesTbl.get(k).nodes.size() <= 0) {
- // note that: we cannot directly do the remove, because it will throw
- // ConcurrentModificationException
- toRemove.add(k);
- // System.out.println("have zero items in superitem " + k);
- // this.tableSuperItems.remove(k);
+ // Remove SuperNodes who may not contain any nodes anymore due to pruning
+ for (Iterator<Entry<Integer, SuperNode>> it = this.superNodesTbl.entrySet().iterator(); it.hasNext(); ) {
+ Entry<Integer, SuperNode> entry = it.next();
+ if (entry.getValue().nodes.isEmpty()) {
+ it.remove();
}
}
- for (Integer t : toRemove) {
- this.superNodesTbl.remove(t);
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/282d5267/src/joshua/decoder/hypergraph/ForestWalker.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/ForestWalker.java b/src/joshua/decoder/hypergraph/ForestWalker.java
index 1838082..72b7fc7 100644
--- a/src/joshua/decoder/hypergraph/ForestWalker.java
+++ b/src/joshua/decoder/hypergraph/ForestWalker.java
@@ -51,7 +51,7 @@ public class ForestWalker {
walk(node, walker, 0);
}
- public void walk(HGNode node, WalkerFunction walker, int nodeIndex) {
+ private void walk(HGNode node, WalkerFunction walker, int nodeIndex) {
// short circuit
if (visitedNodes.contains(node))
return;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/282d5267/src/joshua/decoder/hypergraph/HGNode.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/HGNode.java b/src/joshua/decoder/hypergraph/HGNode.java
index f3f550c..c45f40c 100644
--- a/src/joshua/decoder/hypergraph/HGNode.java
+++ b/src/joshua/decoder/hypergraph/HGNode.java
@@ -54,8 +54,6 @@ public class HGNode {
private Signature signature = null;
// private int hash = 0;
- // For pruning purposes.
- public boolean isDead = false;
protected float score = 0.0f;
// ===============================================================
[02/10] incubator-joshua git commit: Modified KenLM jni to support
querying the lm using strings not only ids. Also added a method to check
whether a word or id is known to the lm. Made output of regression tests more
concise
Posted by mj...@apache.org.
Modified KenLM jni to support querying the lm using strings not only ids. Also added a method to check whether a word or id is known to the lm.
Made output of regression tests more concise
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/cc556006
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/cc556006
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/cc556006
Branch: refs/heads/master
Commit: cc556006e646677f5c0e32755547b2657f59ff92
Parents: d25464c
Author: Felix Hieber <fh...@amazon.com>
Authored: Thu Dec 31 11:57:18 2015 +0100
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Thu Apr 28 15:42:05 2016 -0700
----------------------------------------------------------------------
jni/kenlm_wrap.cc | 57 ++++++++++++++++++++++++++++++--
src/joshua/decoder/ff/lm/KenLM.java | 35 +++++++++++++++++++-
tst/joshua/system/KenLmTest.java | 55 ++++++++++++++++++++++--------
3 files changed, 130 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/cc556006/jni/kenlm_wrap.cc
----------------------------------------------------------------------
diff --git a/jni/kenlm_wrap.cc b/jni/kenlm_wrap.cc
index 6d66f37..16cb54b 100644
--- a/jni/kenlm_wrap.cc
+++ b/jni/kenlm_wrap.cc
@@ -12,7 +12,7 @@
#include <jni.h>
#include <pthread.h>
-// Grr. Everybody's compiler is slightly different and I'm trying to not depend on boost.
+// Grr. Everybody's compiler is slightly different and I'm trying to not depend on boost.
#include <unordered_map>
// Verify that jint and lm::ngram::WordIndex are the same size. If this breaks
@@ -55,7 +55,7 @@ struct Chart {
lm::ngram::ChartState* put(const lm::ngram::ChartState& state) {
uint64_t hashValue = lm::ngram::hash_value(state);
-
+
if (poolHash->find(hashValue) == poolHash->end()) {
lm::ngram::ChartState* pointer = (lm::ngram::ChartState *)pool->Allocate(sizeof(lm::ngram::ChartState));
*pointer = state;
@@ -67,7 +67,7 @@ struct Chart {
};
// Vocab ids above what the vocabulary knows about are unknown and should
-// be mapped to that.
+// be mapped to that.
void MapArray(const std::vector<lm::WordIndex>& map, jint *begin, jint *end) {
for (jint *i = begin; i < end; ++i) {
*i = map[*i];
@@ -88,8 +88,17 @@ public:
virtual ~VirtualBase() {
}
+ // compute/return n-gram probability for array of Joshua word ids
virtual float Prob(jint *begin, jint *end) const = 0;
+ // Compute/return n-gram probability for array of lm:WordIndexes
+ virtual float ProbForWordIndexArray(jint *begin, jint *end) const = 0;
+
+ // Returns the internal lm::WordIndex for a string
+ virtual uint GetLmId(const StringPiece& word) const = 0;
+
+ virtual bool IsKnownWordIndex(const lm::WordIndex& id) const = 0;
+
virtual float ProbRule(jlong *begin, jlong *end, lm::ngram::ChartState& state) const = 0;
virtual float ProbString(jint * const begin, jint * const end,
@@ -131,8 +140,12 @@ public:
}
float Prob(jint * const begin, jint * const end) const {
+ // map Joshua word ids to lm::WordIndexes
MapArray(map_, begin, end);
+ return ProbForWordIndexArray(begin, end);
+ }
+ float ProbForWordIndexArray(jint * const begin, jint * const end) const {
std::reverse(begin, end - 1);
lm::ngram::State ignored;
return m_.FullScoreForgotState(
@@ -141,6 +154,14 @@ public:
ignored).prob;
}
+ uint GetLmId(const StringPiece& word) const {
+ return m_.GetVocabulary().Index(word);
+ }
+
+ bool IsKnownWordIndex(const lm::WordIndex& id) const {
+ return id != m_.GetVocabulary().NotFound();
+ }
+
float ProbRule(jlong * const begin, jlong * const end, lm::ngram::ChartState& state) const {
if (begin == end) return 0.0;
lm::ngram::RuleScore<Model> ruleScore(m_, state);
@@ -353,6 +374,36 @@ JNIEXPORT jfloat JNICALL Java_joshua_decoder_ff_lm_KenLM_prob(
values + length);
}
+JNIEXPORT jfloat JNICALL Java_joshua_decoder_ff_lm_KenLM_probForString(
+ JNIEnv *env, jclass, jlong pointer, jobjectArray arr) {
+ jint length = env->GetArrayLength(arr);
+ if (length <= 0)
+ return 0.0;
+ jint values[length];
+ const VirtualBase* lm_base = reinterpret_cast<const VirtualBase*>(pointer);
+ for (int i=0; i<length; i++) {
+ jstring word = (jstring) env->GetObjectArrayElement(arr, i);
+ const char *str = env->GetStringUTFChars(word, 0);
+ values[i] = lm_base->GetLmId(str);
+ env->ReleaseStringUTFChars(word, str);
+ }
+ return lm_base->ProbForWordIndexArray(values,
+ values + length);
+}
+
+JNIEXPORT jboolean JNICALL Java_joshua_decoder_ff_lm_KenLM_isKnownWord(
+ JNIEnv *env, jclass, jlong pointer, jstring word) {
+ const char *str = env->GetStringUTFChars(word, 0);
+ if (!str)
+ return false;
+ bool ret;
+ const VirtualBase* lm_base = reinterpret_cast<const VirtualBase*>(pointer);
+ lm::WordIndex id = lm_base->GetLmId(str);
+ ret = lm_base->IsKnownWordIndex(id);
+ env->ReleaseStringUTFChars(word, str);
+ return ret;
+}
+
JNIEXPORT jfloat JNICALL Java_joshua_decoder_ff_lm_KenLM_probString(
JNIEnv *env, jclass, jlong pointer, jintArray arr, jint start) {
jint length = env->GetArrayLength(arr);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/cc556006/src/joshua/decoder/ff/lm/KenLM.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/ff/lm/KenLM.java b/src/joshua/decoder/ff/lm/KenLM.java
index be1f1e2..329b631 100644
--- a/src/joshua/decoder/ff/lm/KenLM.java
+++ b/src/joshua/decoder/ff/lm/KenLM.java
@@ -18,6 +18,7 @@
*/
package joshua.decoder.ff.lm;
+import joshua.corpus.Vocabulary;
import joshua.decoder.ff.lm.NGramLanguageModel;
import joshua.decoder.ff.state_maintenance.KenLMState;
@@ -66,6 +67,10 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
private final static native float prob(long ptr, int words[]);
+ private final static native float probForString(long ptr, String[] words);
+
+ private final static native boolean isKnownWord(long ptr, String word);
+
private final static native StateProbPair probRule(long ptr, long pool, long words[]);
private final static native float estimateRule(long ptr, long words[]);
@@ -82,6 +87,16 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
N = order(pointer);
}
+ /**
+ * Constructor if order is not known.
+ * Order will be inferred from the model.
+ */
+ public KenLM(String file_name) {
+ pointer = construct(file_name);
+ N = order(pointer);
+ ngramOrder = N;
+ }
+
public void destroy() {
destroy(pointer);
}
@@ -94,10 +109,17 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
return registerWord(pointer, word, id);
}
- public float prob(int words[]) {
+ public float prob(int[] words) {
return prob(pointer, words);
}
+ /**
+ * Query for n-gram probability using strings.
+ */
+ public float prob(String[] words) {
+ return probForString(pointer, words);
+ }
+
// Apparently Zhifei starts some array indices at 1. Change to 0-indexing.
public float probString(int words[], int start) {
return probString(pointer, words, start - 1);
@@ -146,6 +168,17 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
return estimate;
}
+ /**
+ * The start symbol for a KenLM is the Vocabulary.START_SYM.
+ */
+ public String getStartSymbol() {
+ return Vocabulary.START_SYM;
+ }
+
+ public boolean isKnownWord(String word) {
+ return isKnownWord(pointer, word);
+ }
+
/**
* Inner class used to hold the results returned from KenLM with left-state minimization. Note
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/cc556006/tst/joshua/system/KenLmTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/KenLmTest.java b/tst/joshua/system/KenLmTest.java
index e185aaf..dba74fc 100644
--- a/tst/joshua/system/KenLmTest.java
+++ b/tst/joshua/system/KenLmTest.java
@@ -18,7 +18,9 @@
*/
package joshua.system;
-import static org.junit.Assert.assertEquals;
+import static joshua.corpus.Vocabulary.registerLanguageModel;
+import static joshua.corpus.Vocabulary.unregisterLanguageModels;
+import static org.junit.Assert.*;
import joshua.corpus.Vocabulary;
import joshua.decoder.Decoder;
import joshua.decoder.JoshuaConfiguration;
@@ -29,22 +31,23 @@ import org.junit.Before;
import org.junit.Test;
/**
- * Integration test for KenLM integration into Joshua This test will setup a
- * Joshua instance that loads libkenlm.so
- *
- * @author kellens
+ * KenLM JNI interface tests.
+ * Loads libken.{so,dylib}.
+ * If run in Eclipse, add -Djava.library.path=build/lib to JVM arguments
+ * of the run configuration.
*/
public class KenLmTest {
+ private static final String LANGUAGE_MODEL_PATH = "resources/kenlm/oilers.kenlm";
+
@Test
- public void givenKenLmUsed_whenTranslationsCalled_thenVerifyJniWithSampleCall() {
+ public void givenKenLm_whenQueryingForNgramProbability_thenProbIsCorrect() {
// GIVEN
- String languageModelPath = "resources/kenlm/oilers.kenlm";
+ KenLM kenLm = new KenLM(3, LANGUAGE_MODEL_PATH);
+ int[] words = Vocabulary.addAll("Wayne Gretzky");
+ registerLanguageModel(kenLm);
// WHEN
- KenLM kenLm = new KenLM(3, languageModelPath);
- Vocabulary.registerLanguageModel(kenLm);
- int[] words = Vocabulary.addAll("Wayne Gretzky");
float probability = kenLm.prob(words);
// THEN
@@ -52,15 +55,41 @@ public class KenLmTest {
Float.MIN_VALUE);
}
+ @Test
+ public void givenKenLm_whenQueryingForNgramProbability_thenIdAndStringMethodsReturnTheSame() {
+ // GIVEN
+ KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
+ registerLanguageModel(kenLm);
+ String sentence = "Wayne Gretzky";
+ String[] words = sentence.split("\\s+");
+ int[] ids = Vocabulary.addAll(sentence);
+
+ // WHEN
+ float prob_string = kenLm.prob(words);
+ float prob_id = kenLm.prob(ids);
+
+ // THEN
+ assertEquals("ngram probabilities differ for word and id based n-gram query", prob_string, prob_id,
+ Float.MIN_VALUE);
+
+ }
+
+ @Test
+ public void givenKenLm_whenIsKnownWord_thenReturnValuesAreCorrect() {
+ KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
+ assertTrue(kenLm.isKnownWord("Wayne"));
+ assertFalse(kenLm.isKnownWord("Wayne2222"));
+ }
+
@Before
public void setUp() throws Exception {
Vocabulary.clear();
- Vocabulary.unregisterLanguageModels();
+ unregisterLanguageModels();
}
-
+
@After
public void tearDown() throws Exception {
Vocabulary.clear();
- Vocabulary.unregisterLanguageModels();
+ unregisterLanguageModels();
}
}
[07/10] incubator-joshua git commit: removed string-based
HypothesisExtractor, fixed test-case outputs
Posted by mj...@apache.org.
removed string-based HypothesisExtractor, fixed test-case outputs
The phrase-based decoder is built on hypergraphs and is fine to use the new int-based extractors
Also, phrase-based extraction had use-unique-nbest hard-coded to true, removed this. Fixed test cases by removing alignment string from output, "use_align_index" no longer works at all
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/2fe4ad63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/2fe4ad63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/2fe4ad63
Branch: refs/heads/master
Commit: 2fe4ad63a38186724c1543081471d7a4cbf5c7d2
Parents: 62072fe
Author: Matt Post <po...@cs.jhu.edu>
Authored: Fri Apr 29 00:30:37 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Fri Apr 29 00:31:38 2016 -0400
----------------------------------------------------------------------
.../decoder/hypergraph/KBestExtractor.java | 143 ++---
test/decoder/phrase/constrained/config | 3 +-
test/decoder/phrase/constrained/output.gold | 10 +-
test/decoder/phrase/decode/config | 2 +-
test/decoder/phrase/decode/config.packed | 2 +-
test/decoder/phrase/decode/output.gold | 2 +-
.../phrase/unique-hypotheses/joshua.config | 4 +-
.../phrase/unique-hypotheses/output.gold | 600 +++++++++----------
8 files changed, 373 insertions(+), 393 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/src/joshua/decoder/hypergraph/KBestExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/KBestExtractor.java b/src/joshua/decoder/hypergraph/KBestExtractor.java
index 9107cea..a41ee66 100644
--- a/src/joshua/decoder/hypergraph/KBestExtractor.java
+++ b/src/joshua/decoder/hypergraph/KBestExtractor.java
@@ -31,8 +31,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.PriorityQueue;
-import java.util.Stack;
-import java.util.regex.Matcher;
import joshua.corpus.Vocabulary;
import joshua.decoder.BLEU;
@@ -95,7 +93,6 @@ import joshua.util.FormatUtils;
public class KBestExtractor {
private final JoshuaConfiguration joshuaConfiguration;
private final String outputFormat;
- private final boolean stackDecoding;
private final HashMap<HGNode, VirtualNode> virtualNodesTable = new HashMap<HGNode, VirtualNode>();
// static final String rootSym = JoshuaConfiguration.goal_symbol;
@@ -135,7 +132,6 @@ public class KBestExtractor {
this.joshuaConfiguration = joshuaConfiguration;
this.outputFormat = this.joshuaConfiguration.outputFormat;
- this.stackDecoding = this.joshuaConfiguration.search_algorithm.equals("stack");
this.extractUniqueNbest = joshuaConfiguration.use_unique_nbest;
this.weights = weights;
@@ -177,9 +173,10 @@ public class KBestExtractor {
// DerivationState derivationState = getKthDerivation(node, k);
if (derivationState != null) {
// ==== read the kbest from each hgnode and convert to output format
- String hypothesis = unescapeSpecialSymbols(
- removeSentenceMarkers(
- derivationState.getHypothesis()));
+ String hypothesis = formatForOutput(
+ unescapeSpecialSymbols(
+ removeSentenceMarkers(
+ derivationState.getHypothesis())), derivationState);
/*
* To save space, the decoder only stores the model cost,
@@ -193,8 +190,8 @@ public class KBestExtractor {
outputString = outputFormat
.replace("%k", Integer.toString(k))
- .replace("%s", hypothesis)
- .replace("%S", DeNormalize.processSingleLine(hypothesis))
+ .replace("%s", formatForOutput(hypothesis, derivationState))
+ .replace("%S", DeNormalize.processSingleLine(formatForOutput(hypothesis, derivationState)))
// TODO (kellens): Fix the recapitalization here
.replace("%i", Integer.toString(sentence.id()))
.replace("%f", joshuaConfiguration.moses ? features.mosesString() : features.toString())
@@ -215,7 +212,7 @@ public class KBestExtractor {
/* %a causes output of word level alignments between input and output hypothesis */
if (outputFormat.contains("%a")) {
- outputString = outputString.replace("%a", derivationState.getWordAlignment());
+ outputString = outputString.replace("%a", derivationState.getWordAlignmentString());
}
}
@@ -226,6 +223,52 @@ public class KBestExtractor {
// =========================== end kbestHypergraph
/**
+ * If requested, projects source-side lettercase to target, and appends the alignment from
+ * to the source-side sentence in ||s.
+ *
+ * @param hypothesis
+ * @param state
+ * @return
+ */
+ private String formatForOutput(String hypothesis, DerivationState state) {
+ String output = hypothesis;
+
+ if (joshuaConfiguration.project_case) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = state.getWordAlignment();
+ for (int i = 0; i < points.size(); i++) {
+ List<Integer> target = points.get(i);
+ for (int source: target) {
+ Token token = sentence.getTokens().get(source + 1); // skip <s>
+ String annotation = "";
+ if (token != null && token.getAnnotation("lettercase") != null)
+ annotation = token.getAnnotation("lettercase");
+ if (source != 0 && annotation.equals("upper"))
+ tokens[i] = FormatUtils.capitalize(tokens[i]);
+ else if (annotation.equals("all-upper"))
+ tokens[i] = tokens[i].toUpperCase();
+ }
+ }
+
+ output = String.join(" ", tokens);
+ }
+
+ if (joshuaConfiguration.include_align_index) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = state.getWordAlignment();
+ for (int i = 0; i < tokens.length; i++) {
+ if (i < points.size()) {
+ tokens[i] += String.format(" %d-%d", points.get(i).get(0),
+ points.get(i).get(points.get(i).size()-1));
+ }
+ }
+ output = String.join(" ", tokens);
+ }
+
+ return output;
+ }
+
+ /**
* Convenience function for k-best extraction that prints to STDOUT.
*/
public void lazyKBestExtractOnHG(HyperGraph hg, int topN) throws IOException {
@@ -732,9 +775,15 @@ public class KBestExtractor {
return visitor;
}
- private String getWordAlignment() {
+ private String getWordAlignmentString() {
return visit(new WordAlignmentExtractor()).toString();
}
+
+ private List<List<Integer>> getWordAlignment() {
+ WordAlignmentExtractor extractor = new WordAlignmentExtractor();
+ visit(extractor);
+ return extractor.getFinalWordAlignments();
+ }
private String getTree() {
return visit(new TreeExtractor()).toString();
@@ -751,11 +800,7 @@ public class KBestExtractor {
* that is correct also for Side.SOURCE cases.
*/
private String getHypothesis(final Side side) {
- if (stackDecoding) {
- return visit(new HypothesisExtractor(side)).toString();
- } else {
- return visit(new OutputStringExtractor(side.equals(Side.SOURCE))).toString();
- }
+ return visit(new OutputStringExtractor(side.equals(Side.SOURCE))).toString();
}
private FeatureVector getFeatures() {
@@ -828,72 +873,6 @@ public class KBestExtractor {
}
/**
- * String-based HypothesisExtractor that works with Joshua's Phrase decoder.
- * It is slower than an int[]-based extraction due to many String operations.
- * It also contains a bug for Hiero KBest with NT-reordering rules when
- * the source side is extracted (due to its dependency on target-side ordered
- * hypergraph traversal.
- */
- public class HypothesisExtractor implements DerivationVisitor {
-
- private Side side;
- private Stack<String> outputs;
-
- public HypothesisExtractor(Side side) {
- this.side = side;
- outputs = new Stack<String>();
- }
-
- String ntMatcher = ".*" + Rule.NT_REGEX + ".*";
-
- void merge(String words) {
- if (!words.matches(ntMatcher) && outputs.size() > 0 && outputs.peek().matches(ntMatcher)) {
- String parentWords = outputs.pop();
- String replaced = parentWords.replaceFirst(Rule.NT_REGEX, Matcher.quoteReplacement(words));
-
- merge(replaced);
- } else {
- outputs.add(words);
- }
- }
-
- @Override
- /**
- * Whenever we reach a rule in the depth-first derivaiton, we add it to the stack
- * via a call to the merge() function.
- */
- public void before(DerivationState state, int level, int tailNodeIndex) {
- Rule rule = state.edge.getRule();
- if (rule != null)
- if (side == Side.TARGET) {
- // Output the alignment Moses-style, skipping <s> and </s>, and converting to indices
- // in original sentence
- String alignment = "";
- if (joshuaConfiguration.include_align_index
- && state.parentNode.j != sentence.length() && state.parentNode.j != 1) {
- int i = state.parentNode.j - 1 - state.edge.getRule().getFrench().length + state.edge.getRule().getArity();
- alignment = String.format(" |%d-%d|", i, state.parentNode.j-2);
- }
- merge(String.format("%s%s", state.edge.getRule().getEnglishWords(), alignment));
- } else
- merge(state.edge.getRule().getFrenchWords());
-
- }
-
- @Override
- public void after(DerivationState state, int level, int tailNodeIndex) {
- }
-
- /**
- * After all rules in the grammar have been merged, there should be one item on the stack, which
- * is the complete target (or source) string.
- */
- public String toString() {
- return outputs.pop().replaceAll("<s> ", "").replace(" </s>", "");
- }
- }
-
- /**
* Assembles a Penn treebank format tree for a given derivation.
*/
public class TreeExtractor implements DerivationVisitor {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/constrained/config
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/constrained/config b/test/decoder/phrase/constrained/config
index 3979bd6..be45e0a 100644
--- a/test/decoder/phrase/constrained/config
+++ b/test/decoder/phrase/constrained/config
@@ -8,8 +8,9 @@ top-n = 5
output-format = %i ||| %s ||| %f ||| %c
-include-align-index = true
+include-align-index = false
reordering-limit = 10
+use-unique-nbest = false
# And these are the feature functions to activate.
feature-function = OOVPenalty
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/constrained/output.gold
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/constrained/output.gold b/test/decoder/phrase/constrained/output.gold
index 238387c..a784043 100644
--- a/test/decoder/phrase/constrained/output.gold
+++ b/test/decoder/phrase/constrained/output.gold
@@ -1,5 +1,5 @@
-0 ||| President Obama |8-8| to |7-7| hinder |4-4| a strategy |0-1| for |3-3| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-15.792 tm_pt_1=-17.550 tm_pt_2=-14.599 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=7.000 ||| -15.163
-0 ||| President Obama |8-8| to |7-7| hinder |4-4| a |0-0| strategy |1-1| for |3-3| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.919 tm_pt_1=-17.550 tm_pt_2=-14.917 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=8.000 ||| -15.505
-0 ||| President Obama |8-8| to hinder |3-4| a strategy |0-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-14.986 tm_pt_1=-17.951 tm_pt_2=-14.075 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=6.000 ||| -15.762
-0 ||| President Obama |8-8| to hinder |3-4| a |0-0| strategy |1-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.112 tm_pt_1=-17.951 tm_pt_2=-14.393 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.103
-0 ||| President Obama |8-8| to |3-3| hinder |4-4| a strategy |0-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.329 tm_pt_1=-17.951 tm_pt_2=-15.136 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.257
+0 ||| President Obama to hinder a strategy for Republican re @-@ election ||| tm_pt_0=-15.792 tm_pt_1=-17.550 tm_pt_2=-14.599 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=7.000 ||| -15.163
+0 ||| President Obama to hinder a strategy for Republican re @-@ election ||| tm_pt_0=-16.919 tm_pt_1=-17.550 tm_pt_2=-14.917 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=8.000 ||| -15.505
+0 ||| President Obama to hinder a strategy for Republican re @-@ election ||| tm_pt_0=-14.986 tm_pt_1=-17.951 tm_pt_2=-14.075 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=6.000 ||| -15.762
+0 ||| President Obama to hinder a strategy for Republican re @-@ election ||| tm_pt_0=-16.112 tm_pt_1=-17.951 tm_pt_2=-14.393 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.103
+0 ||| President Obama to hinder a strategy for Republican re @-@ election ||| tm_pt_0=-16.329 tm_pt_1=-17.951 tm_pt_2=-15.136 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.257
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/decode/config
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/decode/config b/test/decoder/phrase/decode/config
index 1edccc5..9987b1a 100644
--- a/test/decoder/phrase/decode/config
+++ b/test/decoder/phrase/decode/config
@@ -9,7 +9,7 @@ top-n = 1
output-format = %i ||| %s ||| %f ||| %c
-include-align-index = true
+include-align-index = false
reordering-limit = 6
# And these are the feature functions to activate.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/decode/config.packed
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/decode/config.packed b/test/decoder/phrase/decode/config.packed
index 1edccc5..9987b1a 100644
--- a/test/decoder/phrase/decode/config.packed
+++ b/test/decoder/phrase/decode/config.packed
@@ -9,7 +9,7 @@ top-n = 1
output-format = %i ||| %s ||| %f ||| %c
-include-align-index = true
+include-align-index = false
reordering-limit = 6
# And these are the feature functions to activate.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/decode/output.gold
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/decode/output.gold b/test/decoder/phrase/decode/output.gold
index 509a3de..0083345 100644
--- a/test/decoder/phrase/decode/output.gold
+++ b/test/decoder/phrase/decode/output.gold
@@ -1 +1 @@
-0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496
+0 ||| a strategy republican to hinder reelection Obama ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/unique-hypotheses/joshua.config
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/unique-hypotheses/joshua.config b/test/decoder/phrase/unique-hypotheses/joshua.config
index 31c6b44..c35b267 100644
--- a/test/decoder/phrase/unique-hypotheses/joshua.config
+++ b/test/decoder/phrase/unique-hypotheses/joshua.config
@@ -5,9 +5,9 @@ lm = kenlm 5 true false 100 lm.1.gz
mark-oovs = false
pop-limit = 100
top-n = 300
-use-unique-nbest = false
+use-unique-nbest = true
output-format = %s
-include-align-index = true
+include-align-index = false
feature-function = OOVPenalty
feature-function = WordPenalty
feature_function = Distortion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/2fe4ad63/test/decoder/phrase/unique-hypotheses/output.gold
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/unique-hypotheses/output.gold b/test/decoder/phrase/unique-hypotheses/output.gold
index 4c3e6b6..0e5fb98 100644
--- a/test/decoder/phrase/unique-hypotheses/output.gold
+++ b/test/decoder/phrase/unique-hypotheses/output.gold
@@ -1,300 +1,300 @@
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct the |3-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct the |3-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to obstruct the |3-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election |5-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct the |4-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| of Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election |5-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election |5-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct the |4-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| obstructing |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| hinder |4-4| for |3-3| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| obstructing |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| obstruct |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| the |5-5| hinder |4-4| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| hinder |4-4| for |3-3| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election |5-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| obstructing |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| obstruct |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| reelection |6-6| the |5-5| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the re @-@ election of |5-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstructing |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election |5-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering the |4-5| re @-@ election of |6-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct the |3-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| hinder |4-4| for |3-3| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the reelection |5-6| of |7-7| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| obstruct |4-4| for |3-3| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstructing |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| obstruct |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstructing |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| obstruct |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| Obama |8-8| of |7-7|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| the |5-5| hinder |4-4| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| hinder |4-4| for |3-3| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the |5-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| reelection |6-6| of |7-7| the |5-5| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstructing |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| hinder |4-4| for |3-3| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| obstruct |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| obstruct |4-4| for |3-3| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| reelection |6-6| the |5-5| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct the |4-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| reelection |6-6| the |5-5| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| of |7-7| reelection |6-6| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| to obstruct the |3-5| re @-@ election |6-6| of Obama |7-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hinder |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election |5-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering the |4-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| reelection |5-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| reelection |6-6| Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the reelection |5-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| obstruct |4-4| for |3-3| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hinder the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to obstruct the |3-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| of Obama |7-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstructing |4-4| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| obstruct |4-4| the re @-@ election of |5-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| reelection |6-6| Obama |8-8| of |7-7|
-a strategy |0-1| republican |2-2| for |3-3| obstruct the |4-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| Obama |8-8| of |7-7|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| Obama |8-8| of |7-7|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hindering |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| reelection |6-6| of |7-7| the |5-5| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| hinder |4-4| for |3-3| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| obstruct |4-4| for |3-3| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| hamper |4-4| for |3-3| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election |6-6| of Obama |7-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| reelection |6-6| of |7-7| the |5-5| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hinder |4-4| the re @-@ election of |5-7| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| obstruct |4-4| for |3-3| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the |5-5| reelection |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct the |4-5| reelection |6-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| reelection |6-6| the |5-5| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| of |7-7| reelection |6-6| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstructing |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder the |4-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| obstruct |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder the |4-5| of |7-7| reelection |6-6| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| of |7-7| reelection |6-6| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct the |4-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| of |7-7| Obama |8-8| reelection |6-6|
-a |0-0| strategy |1-1| republican |2-2| to obstruct |3-4| the reelection |5-6| of |7-7| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to hinder |3-4| the re @-@ election |5-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| hamper |4-4| for |3-3| the |5-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| hinder |4-4| for |3-3| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| reelection |5-6| Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| for |3-3| hindering |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| reelection |6-6| the |5-5| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hamper |4-4| the re @-@ election of |5-7| Obama |8-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| reelection |5-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the reelection |5-6| Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| reelection |6-6| of Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election |6-6| of Obama |7-8|
-a strategy |0-1| republican |2-2| to obstruct |3-4| the |5-5| reelection |6-6| Obama |7-8|
-a strategy |0-1| republican |2-2| to |3-3| hinder |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| obstruct |4-4| the reelection |5-6| of Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| reelection |5-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct the |4-5| re @-@ election |6-6| of Obama |7-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hamper the |4-5| reelection |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| reelection |6-6| Obama |8-8| of |7-7|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| reelection |6-6| Obama |8-8| of |7-7|
-a strategy |0-1| republican |2-2| for |3-3| hinder |4-4| of |7-7| the |5-5| reelection |6-6| Obama |8-8|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the reelection |5-6| Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| hamper |4-4| for |3-3| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| reelection |5-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| reelection |6-6| of |7-7| the |5-5| Obama |8-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| hamper |4-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| obstruct |4-4| for |3-3| the re @-@ election of |5-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| reelection |6-6| of Obama |7-8|
-strategy |1-1| a |0-0| republican |2-2| for |3-3| obstruct |4-4| the re @-@ election of |5-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| to obstruct the |3-5| reelection |6-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the reelection |5-6| of |7-7| Obama |8-8|
-an |0-0| strategy |1-1| republican |2-2| to hinder |3-4| the |5-5| re @-@ election |6-6| of |7-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstructing |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| hinder the |4-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| to |3-3| obstruct |4-4| the |5-5| re @-@ election of |6-7| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder the |4-5| of |7-7| reelection |6-6| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct |4-4| the |5-5| of |7-7| reelection |6-6| Obama |8-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| obstruct the |4-5| reelection |6-6| of Obama |7-8|
-a |0-0| strategy |1-1| republican |2-2| for |3-3| hinder |4-4| the |5-5| of |7-7| Obama |8-8| reelection |6-6|
-a strategy |0-1| republican |2-2| for |3-3| hamper |4-4| the |5-5| reelection |6-6| Obama |8-8| of |7-7|
+a strategy republican for hinder the re @-@ election of Obama
+a strategy republican for hinder the reelection of Obama
+a strategy republican for obstruct the re @-@ election of Obama
+a strategy republican for obstruct the reelection of Obama
+a strategy republican for hamper the re @-@ election of Obama
+a strategy republican for hamper the reelection of Obama
+a strategy republican to obstruct the re @-@ election of Obama
+a strategy republican for hinder reelection of Obama
+a strategy republican to obstruct the reelection of Obama
+a strategy republican to hinder the re @-@ election of Obama
+a strategy republican to hinder the reelection of Obama
+a strategy republican for obstruct reelection of Obama
+a strategy republican for hinder the reelection Obama
+a strategy republican for hamper reelection of Obama
+a strategy republican for hindering the re @-@ election of Obama
+a strategy republican for obstruct the reelection Obama
+a strategy republican for hindering the reelection of Obama
+an strategy republican for hinder the re @-@ election of Obama
+a strategy republican for hamper the reelection Obama
+an strategy republican for hinder the reelection of Obama
+a strategy republican for obstructing the re @-@ election of Obama
+a strategy republican to hinder reelection of Obama
+a strategy republican hinder for the re @-@ election of Obama
+a strategy republican for obstructing the reelection of Obama
+an strategy republican for obstruct the re @-@ election of Obama
+a strategy republican for the hinder reelection of Obama
+a strategy republican hinder for the reelection of Obama
+a strategy republican for hinder reelection the of Obama
+an strategy republican for obstruct the reelection of Obama
+a strategy republican for hinder reelection Obama
+a strategy republican to obstruct the reelection Obama
+strategy a republican for hinder the re @-@ election of Obama
+a strategy republican obstruct for the re @-@ election of Obama
+a strategy republican for hinder the reelection Obama of
+a strategy republican to hinder the reelection Obama
+a strategy republican for hinder reelection of the Obama
+strategy a republican for hinder the reelection of Obama
+a strategy republican obstruct for the reelection of Obama
+an strategy republican for hamper the re @-@ election of Obama
+a strategy republican for obstruct reelection the of Obama
+a strategy republican for hinder the of reelection Obama
+a strategy republican to obstruct reelection of Obama
+a strategy republican for obstruct reelection Obama
+an strategy republican for hamper the reelection of Obama
+strategy a republican for obstruct the re @-@ election of Obama
+a strategy republican for obstruct the reelection Obama of
+a strategy republican hamper for the re @-@ election of Obama
+a strategy republican for obstruct reelection of the Obama
+strategy a republican for obstruct the reelection of Obama
+a strategy republican for obstruct the of reelection Obama
+a strategy republican for hinder the of Obama reelection
+a strategy republican for hindering reelection of Obama
+a strategy republican hamper for the reelection of Obama
+a strategy republican for hamper reelection the of Obama
+an strategy republican to obstruct the re @-@ election of Obama
+a strategy republican for hamper reelection Obama
+a strategy republican for hinder of the reelection Obama
+an strategy republican for hinder reelection of Obama
+strategy a republican for hamper the re @-@ election of Obama
+an strategy republican to obstruct the reelection of Obama
+an strategy republican to hinder the re @-@ election of Obama
+a strategy republican for hamper the reelection Obama of
+a strategy republican for hindering the reelection Obama
+a strategy republican for hinder the re @-@ election Obama
+a strategy republican for obstruct the of Obama reelection
+a strategy republican for hamper reelection of the Obama
+strategy a republican for hamper the reelection of Obama
+an strategy republican to hinder the reelection of Obama
+one strategy republican for hinder the re @-@ election of Obama
+a strategy republican for obstructing reelection of Obama
+a strategy republican for hamper the of reelection Obama
+a strategy republican for hinder the reelection from Obama
+a strategy republican for hinder reelection the Obama
+one strategy republican for hinder the reelection of Obama
+a strategy republican for obstruct of the reelection Obama
+a strategy republican for hinder the Obama reelection of
+an strategy republican for obstruct reelection of Obama
+a strategy republican for hinder reelection of Obama the
+a strategy republican for the reelection hinder of Obama
+a strategy republican for obstruct the re @-@ election Obama
+an strategy republican for hinder the reelection Obama
+a strategy republican to hinder reelection the of Obama
+a strategy republican for hinder 's reelection Obama
+strategy a republican to obstruct the re @-@ election of Obama
+one strategy republican for obstruct the re @-@ election of Obama
+a strategy republican for obstruct the reelection from Obama
+a strategy republican for hamper the of Obama reelection
+a strategy republican for hinder reelection Obama of
+a strategy republican to hinder reelection Obama
+strategy a republican for hinder reelection of Obama
+a strategy republican to obstruct the reelection Obama of
+a strategy republican for obstruct reelection the Obama
+strategy a republican to obstruct the reelection of Obama
+one strategy republican for obstruct the reelection of Obama
+strategy a republican to hinder the re @-@ election of Obama
+a strategy republican for obstructing the reelection Obama
+a strategy republican for obstruct the Obama reelection of
+a strategy republican for obstruct reelection of Obama the
+a strategy republican to obstruct the of reelection Obama
+a strategy republican to hinder the reelection Obama of
+a strategy republican for the hinder reelection Obama
+a strategy republican hinder for the reelection Obama
+a strategy republican for hamper of the reelection Obama
+a strategy republican to hinder reelection of the Obama
+an strategy republican for hamper reelection of Obama
+a strategy republican for the reelection obstruct of Obama
+strategy a republican to hinder the reelection of Obama
+an strategy republican for hindering the re @-@ election of Obama
+an strategy republican for obstruct the reelection Obama
+a strategy republican for hinder the reelection Obama 's
+a strategy republican to hinder the of reelection Obama
+a strategy republican for obstruct 's reelection Obama
+a strategy republican for hinder the reelection for Obama
+a strategy republican for hamper the re @-@ election Obama
+a strategy republican for obstruct reelection Obama of
+an strategy republican for hindering the reelection of Obama
+strategy a republican for obstruct reelection of Obama
+one strategy republican for hamper the re @-@ election of Obama
+a strategy republican for hamper the reelection from Obama
+a strategy republican for hinder the Obama of reelection
+a strategy republican for hinder the Obama reelection
+a strategy republican for the re @-@ election of Obama hinder
+a strategy republican to obstruct reelection the of Obama
+a strategy republican for hinder of Obama the reelection
+a strategy republican to obstruct the of Obama reelection
+strategy a republican for hinder the reelection Obama
+a strategy republican for hamper reelection the Obama
+a strategy republican obstruct for the reelection Obama
+one strategy republican for hamper the reelection of Obama
+a strategy republican for hamper the Obama reelection of
+a strategy republican for hamper reelection of Obama the
+a strategy republican for the reelection of Obama hinder
+a strategy republican to obstruct reelection Obama
+a strategy republican for obstruct the reelection Obama 's
+a strategy republican to hinder the of Obama reelection
+a strategy republican for obstruct the reelection for Obama
+an strategy republican for hamper the reelection Obama
+a strategy republican for hinder reelection Obama of the
+a strategy republican for hamper 's reelection Obama
+a strategy republican to obstruct reelection of the Obama
+a strategy republican for obstruct the Obama of reelection
+a strategy republican for hamper reelection Obama of
+a strategy republican for obstruct the Obama reelection
+a strategy republican for hinder 's re @-@ election Obama
+a strategy republican for the re @-@ election of Obama obstruct
+an strategy republican for obstructing the re @-@ election of Obama
+strategy a republican for hamper reelection of Obama
+a strategy republican to hinder of the reelection Obama
+a strategy republican for obstruct of Obama the reelection
+a strategy republican to obstruct the re @-@ election Obama
+strategy a republican for hindering the re @-@ election of Obama
+strategy a republican for obstruct the reelection Obama
+an strategy republican to hinder reelection of Obama
+a strategy republican for hindering reelection the of Obama
+a strategy republican for hinder reelection from Obama
+an strategy republican hinder for the re @-@ election of Obama
+strategy an republican for hinder the re @-@ election of Obama
+one strategy republican to obstruct the re @-@ election of Obama
+a strategy republican for the reelection of Obama obstruct
+a strategy republican to obstruct the reelection from Obama
+an strategy republican for obstructing the reelection of Obama
+a strategy republican for hindering the reelection Obama of
+a strategy republican for hinder the re @-@ election Obama of
+a strategy republican for hindering reelection Obama
+a strategy republican hamper for the reelection Obama
+strategy a republican for hindering the reelection of Obama
+a strategy republican to hinder the re @-@ election Obama
+one strategy republican for hinder reelection of Obama
+an strategy republican for the hinder reelection of Obama
+an strategy republican hinder for the reelection of Obama
+strategy an republican for hinder the reelection of Obama
+a strategy republican for hamper the reelection Obama 's
+a strategy republican for hindering the of reelection Obama
+one strategy republican to obstruct the reelection of Obama
+one strategy republican to hinder the re @-@ election of Obama
+an strategy republican for hinder reelection the of Obama
+a strategy republican to hinder the reelection from Obama
+a strategy republican for hinder re @-@ election of the Obama
+a strategy republican to obstruct the Obama reelection of
+a strategy republican for obstruct reelection Obama of the
+a strategy republican for hamper the reelection for Obama
+a strategy republican for reelection hinder of Obama
+a strategy republican for hinder reelection the Obama of
+a strategy republican for hindering reelection of the Obama
+a strategy republican for obstruct 's re @-@ election Obama
+an strategy republican for hinder reelection Obama
+a strategy republican to hinder reelection the Obama
+a strategy republican hinder the re @-@ election of Obama for
+one strategy republican to hinder the reelection of Obama
+a strategy republican for hinder of reelection the Obama
+a strategy republican to hinder the Obama reelection of
+an strategy republican to obstruct the reelection Obama
+a strategy republican to hinder reelection of Obama the
+a strategy republican for hamper the Obama of reelection
+a strategy republican for hinder the of re @-@ election Obama
+a strategy republican for hamper the Obama reelection
+a strategy republican for obstruct reelection from Obama
+an strategy republican obstruct for the re @-@ election of Obama
+a strategy republican for the re @-@ election of Obama hamper
+strategy an republican for obstruct the re @-@ election of Obama
+a strategy republican for obstructing reelection the of Obama
+a strategy republican for hinder of Obama reelection the
+a strategy republican for reelection of Obama hinder the
+a strategy republican for hamper of Obama the reelection
+a strategy republican hinder the reelection of Obama for
+strategy a republican for hamper the reelection Obama
+a strategy republican for obstruct the re @-@ election Obama of
+an strategy republican for hinder the reelection Obama of
+an strategy republican to hinder the reelection Obama
+an strategy republican for hinder reelection of the Obama
+one strategy republican for obstruct reelection of Obama
+a strategy republican for obstructing reelection Obama
+an strategy republican obstruct for the reelection of Obama
+a strategy republican for the reelection of Obama hamper
+strategy an republican for obstruct the reelection of Obama
+a strategy republican to hinder 's reelection Obama
+a strategy republican to obstruct of the reelection Obama
+a strategy republican for hindering the of Obama reelection
+an strategy republican for obstruct reelection the of Obama
+a strategy republican for obstruct re @-@ election of the Obama
+an strategy republican for hinder the of reelection Obama
+strategy a republican for obstructing the re @-@ election of Obama
+an strategy republican to obstruct reelection of Obama
+a strategy republican to hinder reelection Obama of
+a strategy republican for reelection obstruct of Obama
+strategy a republican to hinder reelection of Obama
+one strategy republican for hinder the reelection Obama
+a strategy republican for obstruct reelection the Obama of
+a strategy republican to the reelection hinder of Obama
+strategy a republican hinder for the re @-@ election of Obama
+an strategy republican for obstruct reelection Obama
+a strategy republican for obstructing the reelection Obama of
+a strategy republican for hinder reelection Obama the
+a strategy republican for hinder reelection Obama 's
+a strategy republican obstruct the re @-@ election of Obama for
+a strategy republican for hamper reelection Obama of the
+a strategy republican for reelection of hinder the Obama
+a strategy republican for obstructing reelection of the Obama
+a strategy republican for obstruct of reelection the Obama
+strategy a republican for obstructing the reelection of Obama
+a strategy republican to obstruct the reelection Obama 's
+a strategy republican for obstruct the of re @-@ election Obama
+a strategy republican for hinder reelection for Obama
+a strategy republican for hamper 's re @-@ election Obama
+a strategy republican for the hinder reelection Obama of
+a strategy republican hinder for the reelection Obama of
+a strategy republican for hinder of the re @-@ election Obama
+a strategy republican for the reelection hinder Obama
+a strategy republican to obstruct the reelection for Obama
+a strategy republican for hinder reelection Obama the of
+a strategy republican for obstructing the of reelection Obama
+a strategy republican for obstruct of Obama reelection the
+strategy a republican for the hinder reelection of Obama
+strategy a republican hinder for the reelection of Obama
+a strategy republican obstruct the reelection of Obama for
+a strategy republican for the reelection of hinder Obama
+an strategy republican for obstruct the reelection Obama of
+a strategy republican for hamper reelection from Obama
+an strategy republican hamper for the re @-@ election of Obama
+strategy a republican for hinder reelection the of Obama
+a strategy republican to hinder the reelection Obama 's
+an strategy republican for obstruct reelection of the Obama
+strategy an republican for hamper the re @-@ election of Obama
+a strategy republican hinder for the of reelection Obama
+a strategy republican to obstruct reelection the Obama
+a strategy republican to obstruct the Obama of reelection
+a strategy republican to hinder the reelection for Obama
+a strategy republican for hamper the re @-@ election Obama of
+a strategy republican for hindering the re @-@ election Obama
+a strategy republican to obstruct the Obama reelection
+an strategy republican for obstruct the of reelection Obama
+a strategy republican for hindering of the reelection Obama
+a strategy republican to obstruct reelection of Obama the
+strategy a republican for hinder reelection Obama
+an strategy republican for hinder the of Obama reelection
+one strategy republican for hamper reelection of Obama
+an strategy republican for hindering reelection of Obama
+an strategy republican hamper for the reelection of Obama
+one strategy republican for hindering the re @-@ election of Obama
+strategy a republican to obstruct the reelection Obama
+strategy an republican for hamper the reelection of Obama
+one strategy republican for obstruct the reelection Obama
+a strategy republican for hindering the reelection from Obama
+a strategy republican to the reelection obstruct of Obama
+a strategy republican for reelection of Obama hinder
+a strategy republican for hinder of Obama reelection
+an strategy republican for hamper reelection the of Obama
+strategy a republican obstruct for the re @-@ election of Obama
+a strategy republican for hamper re @-@ election of the Obama
+a strategy republican for obstruct reelection Obama the
+a strategy republican for obstruct reelection Obama 's
+a strategy republican to hinder the Obama of reelection
+a strategy republican to hinder the Obama reelection
+a strategy for hinder the re @-@ election of Obama republican
+a strategy republican for obstruct reelection for Obama
+strategy a republican for hinder the reelection Obama of
+a strategy republican for hamper reelection the Obama of
+a strategy republican obstruct for the reelection Obama of
+a strategy republican for obstruct of the re @-@ election Obama
+one strategy republican for hindering the reelection of Obama
[09/10] incubator-joshua git commit: removed non-functioning
include_align_index code, added test case in case it's restored
Posted by mj...@apache.org.
removed non-functioning include_align_index code, added test case in case it's restored
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/a6bf6283
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/a6bf6283
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/a6bf6283
Branch: refs/heads/master
Commit: a6bf6283aedfd4b3538bc36cd9264c9b610281ff
Parents: 73523c5
Author: Matt Post <po...@cs.jhu.edu>
Authored: Fri Apr 29 00:45:09 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Fri Apr 29 00:45:09 2016 -0400
----------------------------------------------------------------------
.../decoder/hypergraph/KBestExtractor.java | 25 +++-------
test/decoder/phrase/include-align-index/README | 2 +
test/decoder/phrase/include-align-index/config | 29 +++++++++++
.../phrase/include-align-index/corpus.es | 1 +
test/decoder/phrase/include-align-index/lm.1.gz | Bin 0 -> 2235 bytes
test/decoder/phrase/include-align-index/log | 50 +++++++++++++++++++
test/decoder/phrase/include-align-index/output | 1 +
.../phrase/include-align-index/output.gold | 1 +
.../phrase/include-align-index/rules.1.gz | Bin 0 -> 2998042 bytes
test/decoder/phrase/include-align-index/test.sh | 17 +++++++
10 files changed, 108 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/src/joshua/decoder/hypergraph/KBestExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/KBestExtractor.java b/src/joshua/decoder/hypergraph/KBestExtractor.java
index a41ee66..98f6f15 100644
--- a/src/joshua/decoder/hypergraph/KBestExtractor.java
+++ b/src/joshua/decoder/hypergraph/KBestExtractor.java
@@ -166,18 +166,19 @@ public class KBestExtractor {
public String getKthHyp(HGNode node, int k) {
String outputString = null;
-
+
// Determine the k-best hypotheses at each HGNode
VirtualNode virtualNode = getVirtualNode(node);
DerivationState derivationState = virtualNode.lazyKBestExtractOnNode(this, k);
// DerivationState derivationState = getKthDerivation(node, k);
if (derivationState != null) {
// ==== read the kbest from each hgnode and convert to output format
- String hypothesis = formatForOutput(
+ String hypothesis = maybeProjectCase(
unescapeSpecialSymbols(
removeSentenceMarkers(
derivationState.getHypothesis())), derivationState);
+
/*
* To save space, the decoder only stores the model cost,
* no the individual feature values.
@@ -190,8 +191,8 @@ public class KBestExtractor {
outputString = outputFormat
.replace("%k", Integer.toString(k))
- .replace("%s", formatForOutput(hypothesis, derivationState))
- .replace("%S", DeNormalize.processSingleLine(formatForOutput(hypothesis, derivationState)))
+ .replace("%s", hypothesis)
+ .replace("%S", DeNormalize.processSingleLine(hypothesis))
// TODO (kellens): Fix the recapitalization here
.replace("%i", Integer.toString(sentence.id()))
.replace("%f", joshuaConfiguration.moses ? features.mosesString() : features.toString())
@@ -230,9 +231,9 @@ public class KBestExtractor {
* @param state
* @return
*/
- private String formatForOutput(String hypothesis, DerivationState state) {
+ private String maybeProjectCase(String hypothesis, DerivationState state) {
String output = hypothesis;
-
+
if (joshuaConfiguration.project_case) {
String[] tokens = hypothesis.split("\\s+");
List<List<Integer>> points = state.getWordAlignment();
@@ -253,18 +254,6 @@ public class KBestExtractor {
output = String.join(" ", tokens);
}
- if (joshuaConfiguration.include_align_index) {
- String[] tokens = hypothesis.split("\\s+");
- List<List<Integer>> points = state.getWordAlignment();
- for (int i = 0; i < tokens.length; i++) {
- if (i < points.size()) {
- tokens[i] += String.format(" %d-%d", points.get(i).get(0),
- points.get(i).get(points.get(i).size()-1));
- }
- }
- output = String.join(" ", tokens);
- }
-
return output;
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/README
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/README b/test/decoder/phrase/include-align-index/README
new file mode 100644
index 0000000..d0c0813
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/README
@@ -0,0 +1,2 @@
+Added non-functioning example that will test outputting phrase alignments if
+that ability is ever restored.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/config
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/config b/test/decoder/phrase/include-align-index/config
new file mode 100644
index 0000000..f30014d
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/config
@@ -0,0 +1,29 @@
+tm = moses -owner pt -maxspan 0 -path rules.1.gz -max-source-len 5
+feature-function = StateMinimizingLanguageModel -lm_order 5 -lm_file lm.1.gz
+
+search = stack
+
+mark-oovs = false
+pop-limit = 10
+top-n = 1
+
+output-format = %i ||| %s ||| %f ||| %c
+
+include-align-index = true
+reordering-limit = 6
+
+# And these are the feature functions to activate.
+feature-function = OOVPenalty
+feature-function = WordPenalty
+feature-function = Distortion
+feature-function = PhrasePenalty -owner pt
+
+OOVPenalty 1.0
+Distortion 0.114849
+WordPenalty -0.201544
+PhrasePenalty -0.236965
+tm_pt_0 0.0370068
+tm_pt_1 0.0495759
+tm_pt_2 0.196742
+tm_pt_3 0.0745423
+lm_0 0.204412452147565
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/corpus.es
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/corpus.es b/test/decoder/phrase/include-align-index/corpus.es
new file mode 100644
index 0000000..6e255f9
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/corpus.es
@@ -0,0 +1 @@
+una estrategia republicana para obstaculizar la reelección de Obama
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/lm.1.gz
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/lm.1.gz b/test/decoder/phrase/include-align-index/lm.1.gz
new file mode 100644
index 0000000..3f4c453
Binary files /dev/null and b/test/decoder/phrase/include-align-index/lm.1.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/log
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/log b/test/decoder/phrase/include-align-index/log
new file mode 100644
index 0000000..05cd80f
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/log
@@ -0,0 +1,50 @@
+Parameters read from configuration file:
+ tm = 'moses -owner pt -maxspan 0 -path rules.1.gz -max-source-len 5'
+ featurefunction = 'StateMinimizingLanguageModel -lm_order 5 -lm_file lm.1.gz'
+ search = 'stack'
+ markoovs = 'false'
+ poplimit = '10'
+ topn = '1'
+ outputformat = '%i ||| %s ||| %f ||| %c'
+ includealignindex = 'true'
+ reorderinglimit = '6'
+ featurefunction = 'OOVPenalty'
+ featurefunction = 'WordPenalty'
+ featurefunction = 'Distortion'
+ featurefunction = 'PhrasePenalty -owner pt'
+Parameters overridden from the command line:
+ threads = '1'
+ c = 'config'
+Read 9 weights (0 of them dense)
+Reading grammar from file rules.1.gz...
+........10........20........30........40........50........60........70........80........90.....100%
+MemoryBasedBatchGrammar: Read 165161 rules with 18 distinct source sides from 'rules.1.gz'
+Couldn't create a GrammarReader for file null with format phrase
+MemoryBasedBatchGrammar: Read 0 rules with 0 distinct source sides from 'null'
+Memory used 219.6 MB
+Grammar loading took: 0 seconds.
+Stateful object with state index 0
+Loading the LM will be faster if you build a binary file.
+Reading lm.1.gz
+----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
+****************************************************************************************************
+FEATURE: tm_pt (weight 0.000)
+FEATURE: tm_custom (weight 0.000)
+FEATURE: lm_0, order 5 (weight 0.204)
+FEATURE: OOVPenalty (weight 1.000)
+FEATURE: WordPenalty (weight -0.202)
+FEATURE: Distortion (weight 0.115)
+FEATURE: PhrasePenalty (weight -0.237)
+Grammar sorting happening lazily on-demand.
+Model loading took 0 seconds
+Memory used 219.6 MB
+Input 0: <s> una estrategia republicana para obstaculizar la reelección de Obama </s>
+Input 0: Collecting options took 0.000 seconds
+Input 0: Search took 0.013 seconds
+Input 0: Translation took 1.532 seconds
+Input 0: Memory used is 392.5 MB
+Translation 0: -7.496 a strategy republican to hinder reelection Obama
+Input 0: 1-best extraction took 0.026 seconds
+Decoding completed.
+Memory used 401.6 MB
+Total running time: 2 seconds
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/output
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/output b/test/decoder/phrase/include-align-index/output
new file mode 100644
index 0000000..509a3de
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/output
@@ -0,0 +1 @@
+0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/output.gold
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/output.gold b/test/decoder/phrase/include-align-index/output.gold
new file mode 100644
index 0000000..509a3de
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/output.gold
@@ -0,0 +1 @@
+0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/rules.1.gz
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/rules.1.gz b/test/decoder/phrase/include-align-index/rules.1.gz
new file mode 100644
index 0000000..14466e9
Binary files /dev/null and b/test/decoder/phrase/include-align-index/rules.1.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a6bf6283/test/decoder/phrase/include-align-index/test.sh
----------------------------------------------------------------------
diff --git a/test/decoder/phrase/include-align-index/test.sh b/test/decoder/phrase/include-align-index/test.sh
new file mode 100644
index 0000000..4732f73
--- /dev/null
+++ b/test/decoder/phrase/include-align-index/test.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -u
+
+cat corpus.es | $JOSHUA/bin/joshua-decoder -threads 1 -c config > output 2> log
+
+# Compare
+diff -u output output.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output log
+ exit 0
+else
+ exit 1
+fi
+
+
[10/10] incubator-joshua git commit: Merge branch
'structured_translations'
Posted by mj...@apache.org.
Merge branch 'structured_translations'
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/3b2a17c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/3b2a17c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/3b2a17c0
Branch: refs/heads/master
Commit: 3b2a17c0cac41f87df507281fab77bd0c9200e30
Parents: 051cc48 a6bf628
Author: Matt Post <po...@cs.jhu.edu>
Authored: Fri Apr 29 00:46:53 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Fri Apr 29 00:46:53 2016 -0400
----------------------------------------------------------------------
jni/kenlm_wrap.cc | 57 +-
resources/berkeley_lm/lm | 16 +
resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
resources/wa_grammar | 6 +-
src/joshua/decoder/Decoder.java | 4 +
src/joshua/decoder/JoshuaConfiguration.java | 18 +-
src/joshua/decoder/StructuredTranslation.java | 72 +--
src/joshua/decoder/Translation.java | 182 ++----
src/joshua/decoder/chart_parser/Cell.java | 62 +-
src/joshua/decoder/chart_parser/Chart.java | 8 +-
src/joshua/decoder/ff/lm/KenLM.java | 35 +-
src/joshua/decoder/ff/lm/LanguageModelFF.java | 28 +-
.../decoder/hypergraph/AlignedSourceTokens.java | 18 +
.../decoder/hypergraph/AllSpansWalker.java | 4 +-
.../hypergraph/FeatureVectorExtractor.java | 80 +++
src/joshua/decoder/hypergraph/ForestWalker.java | 14 +-
.../GrammarBuilderWalkerFunction.java | 2 +-
src/joshua/decoder/hypergraph/HGNode.java | 2 -
src/joshua/decoder/hypergraph/HyperGraph.java | 10 +-
.../decoder/hypergraph/KBestExtractor.java | 348 ++++-------
.../hypergraph/OutputStringExtractor.java | 195 ++++++
.../decoder/hypergraph/ViterbiExtractor.java | 159 ++---
.../ViterbiFeatureVectorWalkerFunction.java | 44 --
.../ViterbiOutputStringWalkerFunction.java | 96 ---
.../decoder/hypergraph/WalkerFunction.java | 9 +-
.../hypergraph/WordAlignmentExtractor.java | 32 +-
.../decoder/hypergraph/WordAlignmentState.java | 20 +-
src/joshua/decoder/io/JSONMessage.java | 2 +-
src/joshua/oracle/OracleExtractionHG.java | 6 +-
test/bn-en/hiero/input.bn | 2 +-
test/decoder/lowercaser/output.gold | 2 +
test/decoder/lowercaser/test.sh | 11 +-
test/decoder/phrase/constrained/config | 3 +-
test/decoder/phrase/constrained/output.gold | 10 +-
test/decoder/phrase/decode/config | 2 +-
test/decoder/phrase/decode/config.packed | 2 +-
test/decoder/phrase/decode/output.gold | 2 +-
test/decoder/phrase/include-align-index/README | 2 +
test/decoder/phrase/include-align-index/config | 29 +
.../phrase/include-align-index/corpus.es | 1 +
test/decoder/phrase/include-align-index/lm.1.gz | Bin 0 -> 2235 bytes
test/decoder/phrase/include-align-index/log | 50 ++
test/decoder/phrase/include-align-index/output | 1 +
.../phrase/include-align-index/output.gold | 1 +
.../phrase/include-align-index/rules.1.gz | Bin 0 -> 2998042 bytes
test/decoder/phrase/include-align-index/test.sh | 17 +
.../phrase/unique-hypotheses/joshua.config | 4 +-
.../phrase/unique-hypotheses/output.gold | 600 +++++++++----------
.../decoder/ff/lm/LanguageModelFFTest.java | 94 +++
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 62 ++
.../kbest_extraction/KBestExtractionTest.java | 20 +-
.../ConstrainedPhraseDecodingTest.java | 20 +-
.../phrase/decode/PhraseDecodingTest.java | 20 +-
tst/joshua/system/AlignmentMapTest.java | 20 +-
tst/joshua/system/KenLmTest.java | 75 ++-
.../system/MultithreadedTranslationTests.java | 20 +-
tst/joshua/system/StructuredOutputTest.java | 20 +-
.../system/StructuredTranslationTest.java | 49 +-
tst/joshua/util/FormatUtilsTest.java | 20 +-
61 files changed, 1656 insertions(+), 1032 deletions(-)
----------------------------------------------------------------------
[03/10] incubator-joshua git commit: Updated license files for tests
Posted by mj...@apache.org.
Updated license files for tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d25464cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d25464cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d25464cc
Branch: refs/heads/master
Commit: d25464cc5a5436f5efdd113369c53b3c594fff48
Parents: b68ccaa
Author: Kellen Sunderland <ke...@amazon.com>
Authored: Wed Apr 27 15:12:00 2016 -0700
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Thu Apr 28 15:42:05 2016 -0700
----------------------------------------------------------------------
.../decoder/ff/lm/LanguageModelFFTest.java | 20 +++++++++++++++++++-
.../kbest_extraction/KBestExtractionTest.java | 20 +++++++++++++++++++-
.../ConstrainedPhraseDecodingTest.java | 20 +++++++++++++++++++-
.../phrase/decode/PhraseDecodingTest.java | 20 +++++++++++++++++++-
tst/joshua/system/AlignmentMapTest.java | 20 +++++++++++++++++++-
tst/joshua/system/KenLmTest.java | 20 +++++++++++++++++++-
.../system/MultithreadedTranslationTests.java | 20 +++++++++++++++++++-
tst/joshua/system/StructuredOutputTest.java | 20 +++++++++++++++++++-
.../system/StructuredTranslationTest.java | 20 +++++++++++++++++++-
tst/joshua/util/FormatUtilsTest.java | 20 +++++++++++++++++++-
10 files changed, 190 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java b/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
index 64c9794..83f5397 100644
--- a/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
+++ b/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
@@ -1,4 +1,22 @@
-package joshua.decoder.ff.lm;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.decoder.ff.lm;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/decoder/kbest_extraction/KBestExtractionTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/decoder/kbest_extraction/KBestExtractionTest.java b/tst/joshua/decoder/kbest_extraction/KBestExtractionTest.java
index af6d670..26c503a 100644
--- a/tst/joshua/decoder/kbest_extraction/KBestExtractionTest.java
+++ b/tst/joshua/decoder/kbest_extraction/KBestExtractionTest.java
@@ -1,4 +1,22 @@
-package joshua.decoder.kbest_extraction;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.decoder.kbest_extraction;
import java.io.IOException;
import java.nio.file.Path;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java b/tst/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
index 6508a63..6abfbe2 100644
--- a/tst/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
+++ b/tst/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
@@ -1,4 +1,22 @@
-package joshua.decoder.phrase.constrained;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.decoder.phrase.constrained;
import java.io.IOException;
import java.nio.file.Path;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/decoder/phrase/decode/PhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/decoder/phrase/decode/PhraseDecodingTest.java b/tst/joshua/decoder/phrase/decode/PhraseDecodingTest.java
index 707fa1b..4785aff 100644
--- a/tst/joshua/decoder/phrase/decode/PhraseDecodingTest.java
+++ b/tst/joshua/decoder/phrase/decode/PhraseDecodingTest.java
@@ -1,4 +1,22 @@
-package joshua.decoder.phrase.decode;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.decoder.phrase.decode;
import java.io.IOException;
import java.nio.file.Path;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/system/AlignmentMapTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/AlignmentMapTest.java b/tst/joshua/system/AlignmentMapTest.java
index 7d383a8..50c3aff 100644
--- a/tst/joshua/system/AlignmentMapTest.java
+++ b/tst/joshua/system/AlignmentMapTest.java
@@ -1,4 +1,22 @@
-package joshua.system;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.system;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/system/KenLmTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/KenLmTest.java b/tst/joshua/system/KenLmTest.java
index 8445410..e185aaf 100644
--- a/tst/joshua/system/KenLmTest.java
+++ b/tst/joshua/system/KenLmTest.java
@@ -1,4 +1,22 @@
-package joshua.system;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.system;
import static org.junit.Assert.assertEquals;
import joshua.corpus.Vocabulary;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/system/MultithreadedTranslationTests.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/MultithreadedTranslationTests.java b/tst/joshua/system/MultithreadedTranslationTests.java
index 679b254..b257aa6 100644
--- a/tst/joshua/system/MultithreadedTranslationTests.java
+++ b/tst/joshua/system/MultithreadedTranslationTests.java
@@ -1,4 +1,22 @@
-package joshua.system;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.system;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/system/StructuredOutputTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/StructuredOutputTest.java b/tst/joshua/system/StructuredOutputTest.java
index 981f9d8..12e6e88 100644
--- a/tst/joshua/system/StructuredOutputTest.java
+++ b/tst/joshua/system/StructuredOutputTest.java
@@ -1,4 +1,22 @@
-package joshua.system;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.system;
import java.util.Arrays;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/system/StructuredTranslationTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/StructuredTranslationTest.java b/tst/joshua/system/StructuredTranslationTest.java
index e14fae2..7460614 100644
--- a/tst/joshua/system/StructuredTranslationTest.java
+++ b/tst/joshua/system/StructuredTranslationTest.java
@@ -1,4 +1,22 @@
-package joshua.system;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.system;
import static java.util.Arrays.asList;
import static joshua.decoder.ff.FeatureVector.DENSE_FEATURE_NAMES;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d25464cc/tst/joshua/util/FormatUtilsTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/util/FormatUtilsTest.java b/tst/joshua/util/FormatUtilsTest.java
index da406cb..254522d 100644
--- a/tst/joshua/util/FormatUtilsTest.java
+++ b/tst/joshua/util/FormatUtilsTest.java
@@ -1,4 +1,22 @@
-package joshua.util;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package joshua.util;
import static joshua.util.FormatUtils.cleanNonTerminal;
import static joshua.util.FormatUtils.escapeSpecialSymbols;
[06/10] incubator-joshua git commit: Merge branch 'master' of
github.com:KellenSunderland/incubator-joshua into structured_translations
Posted by mj...@apache.org.
Merge branch 'master' of github.com:KellenSunderland/incubator-joshua into structured_translations
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/62072fe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/62072fe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/62072fe8
Branch: refs/heads/master
Commit: 62072fe80f457f3dd8273b50f528937ac9635250
Parents: 051cc48 282d526
Author: Matt Post <po...@cs.jhu.edu>
Authored: Thu Apr 28 19:49:19 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Apr 28 19:49:19 2016 -0400
----------------------------------------------------------------------
jni/kenlm_wrap.cc | 57 ++++-
resources/berkeley_lm/lm | 16 ++
resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
resources/wa_grammar | 6 +-
src/joshua/decoder/Decoder.java | 4 +
src/joshua/decoder/JoshuaConfiguration.java | 11 +-
src/joshua/decoder/StructuredTranslation.java | 72 ++----
src/joshua/decoder/Translation.java | 182 +++++--------
src/joshua/decoder/chart_parser/Cell.java | 62 ++---
src/joshua/decoder/chart_parser/Chart.java | 8 +-
src/joshua/decoder/ff/lm/KenLM.java | 35 ++-
src/joshua/decoder/ff/lm/LanguageModelFF.java | 28 +-
.../decoder/hypergraph/AlignedSourceTokens.java | 18 ++
.../decoder/hypergraph/AllSpansWalker.java | 4 +-
.../hypergraph/FeatureVectorExtractor.java | 80 ++++++
src/joshua/decoder/hypergraph/ForestWalker.java | 14 +-
.../GrammarBuilderWalkerFunction.java | 2 +-
src/joshua/decoder/hypergraph/HGNode.java | 2 -
src/joshua/decoder/hypergraph/HyperGraph.java | 10 +-
.../decoder/hypergraph/KBestExtractor.java | 254 +++++++------------
.../hypergraph/OutputStringExtractor.java | 195 ++++++++++++++
.../decoder/hypergraph/ViterbiExtractor.java | 159 ++++++------
.../ViterbiFeatureVectorWalkerFunction.java | 44 ----
.../ViterbiOutputStringWalkerFunction.java | 96 -------
.../decoder/hypergraph/WalkerFunction.java | 9 +-
.../hypergraph/WordAlignmentExtractor.java | 32 ++-
.../decoder/hypergraph/WordAlignmentState.java | 20 +-
src/joshua/decoder/io/JSONMessage.java | 2 +-
src/joshua/oracle/OracleExtractionHG.java | 6 +-
test/bn-en/hiero/input.bn | 2 +-
.../decoder/ff/lm/LanguageModelFFTest.java | 94 +++++++
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 62 +++++
.../kbest_extraction/KBestExtractionTest.java | 20 +-
.../ConstrainedPhraseDecodingTest.java | 20 +-
.../phrase/decode/PhraseDecodingTest.java | 20 +-
tst/joshua/system/AlignmentMapTest.java | 20 +-
tst/joshua/system/KenLmTest.java | 75 +++++-
.../system/MultithreadedTranslationTests.java | 20 +-
tst/joshua/system/StructuredOutputTest.java | 20 +-
.../system/StructuredTranslationTest.java | 49 +++-
tst/joshua/util/FormatUtilsTest.java | 20 +-
43 files changed, 1194 insertions(+), 656 deletions(-)
----------------------------------------------------------------------
[05/10] incubator-joshua git commit:
LanguageModelFF.estimateFutureCost refactorings and test
Posted by mj...@apache.org.
LanguageModelFF.estimateFutureCost refactorings and test
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/b68ccaae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/b68ccaae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/b68ccaae
Branch: refs/heads/master
Commit: b68ccaae50139d97bce12c2b3fbc8a0132bcb235
Parents: 5b79128
Author: Pavel Danchenko <da...@amazon.com>
Authored: Tue Dec 22 12:49:22 2015 +0100
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Thu Apr 28 15:42:05 2016 -0700
----------------------------------------------------------------------
resources/berkeley_lm/lm | 16 ++++
resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
src/joshua/decoder/Decoder.java | 4 +
src/joshua/decoder/ff/lm/LanguageModelFF.java | 28 ++++---
.../decoder/ff/lm/LanguageModelFFTest.java | 76 +++++++++++++++++++
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 62 +++++++++++++++
8 files changed, 174 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/resources/berkeley_lm/lm
----------------------------------------------------------------------
diff --git a/resources/berkeley_lm/lm b/resources/berkeley_lm/lm
new file mode 100644
index 0000000..05b4e6b
--- /dev/null
+++ b/resources/berkeley_lm/lm
@@ -0,0 +1,16 @@
+
+\data\
+ngram 1=5
+ngram 2=3
+
+\1-grams:
+-99.000000 <unk>
+-99.000000 <s> -1.752754
+-2.034158 the -0.800943
+-5.318589 chat-rooms -0.151088
+-1.495702 </s>
+
+\2-grams:
+-1.773970 <s> the
+-4.878868 the chat-rooms
+-0.499794 chat-rooms </s>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/resources/berkeley_lm/lm.berkeleylm
----------------------------------------------------------------------
diff --git a/resources/berkeley_lm/lm.berkeleylm b/resources/berkeley_lm/lm.berkeleylm
new file mode 100644
index 0000000..c048464
Binary files /dev/null and b/resources/berkeley_lm/lm.berkeleylm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/resources/berkeley_lm/lm.berkeleylm.gz
----------------------------------------------------------------------
diff --git a/resources/berkeley_lm/lm.berkeleylm.gz b/resources/berkeley_lm/lm.berkeleylm.gz
new file mode 100644
index 0000000..f9f8d16
Binary files /dev/null and b/resources/berkeley_lm/lm.berkeleylm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/resources/berkeley_lm/lm.gz
----------------------------------------------------------------------
diff --git a/resources/berkeley_lm/lm.gz b/resources/berkeley_lm/lm.gz
new file mode 100644
index 0000000..ae47266
Binary files /dev/null and b/resources/berkeley_lm/lm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/src/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/Decoder.java b/src/joshua/decoder/Decoder.java
index 1b12dda..aab6d36 100644
--- a/src/joshua/decoder/Decoder.java
+++ b/src/joshua/decoder/Decoder.java
@@ -530,6 +530,10 @@ public class Decoder {
e.printStackTrace();
}
}
+ resetGlobalState();
+ }
+
+ public static void resetGlobalState() {
// clear/reset static variables
DENSE_FEATURE_NAMES.clear();
Vocabulary.clear();
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/src/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/ff/lm/LanguageModelFF.java b/src/joshua/decoder/ff/lm/LanguageModelFF.java
index 38f1a74..a002de7 100644
--- a/src/joshua/decoder/ff/lm/LanguageModelFF.java
+++ b/src/joshua/decoder/ff/lm/LanguageModelFF.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import com.google.common.primitives.Ints;
+
import joshua.corpus.Vocabulary;
import joshua.decoder.JoshuaConfiguration;
import joshua.decoder.Support;
@@ -341,18 +343,12 @@ public class LanguageModelFF extends StatefulFF {
int[] leftContext = state.getLeftLMStateWords();
if (null != leftContext) {
- List<Integer> words = new ArrayList<Integer>();
- for (int w : leftContext)
- words.add(w);
-
- boolean considerIncompleteNgrams = true;
boolean skipStart = true;
- if (words.get(0) != startSymbolId) {
+ if (leftContext[0] != startSymbolId) {
skipStart = false;
}
- estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+ estimate += scoreChunkLogP(leftContext, true, skipStart);
}
-
return weight * estimate;
}
@@ -476,6 +472,15 @@ public class LanguageModelFF extends StatefulFF {
return new NgramDPState(leftContext, rightContext);
}
+
+ /**
+ * Compatibility method for {@link #scoreChunkLogP(int[], boolean, boolean)}
+ */
+ private float scoreChunkLogP(List<Integer> words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+ return scoreChunkLogP(Ints.toArray(words), considerIncompleteNgrams, skipStart);
+ }
+
/**
* This function is basically a wrapper for NGramLanguageModel::sentenceLogProbability(). It
* computes the probability of a phrase ("chunk"), using lower-order n-grams for the first n-1
@@ -486,11 +491,11 @@ public class LanguageModelFF extends StatefulFF {
* @param skipStart
* @return the phrase log probability
*/
- private float scoreChunkLogP(List<Integer> words, boolean considerIncompleteNgrams,
+ private float scoreChunkLogP(int[] words, boolean considerIncompleteNgrams,
boolean skipStart) {
float score = 0.0f;
- if (words.size() > 0) {
+ if (words.length > 0) {
int startIndex;
if (!considerIncompleteNgrams) {
startIndex = this.ngramOrder;
@@ -499,8 +504,7 @@ public class LanguageModelFF extends StatefulFF {
} else {
startIndex = 1;
}
- score = this.languageModel.sentenceLogProbability(
- Support.subIntArray(words, 0, words.size()), this.ngramOrder, startIndex);
+ score = this.languageModel.sentenceLogProbability(words, this.ngramOrder, startIndex);
}
return score;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java b/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
new file mode 100644
index 0000000..64c9794
--- /dev/null
+++ b/tst/joshua/decoder/ff/lm/LanguageModelFFTest.java
@@ -0,0 +1,76 @@
+package joshua.decoder.ff.lm;
+
+import static org.junit.Assert.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import joshua.corpus.Vocabulary;
+import joshua.decoder.Decoder;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.ff.FeatureVector;
+import joshua.decoder.ff.state_maintenance.NgramDPState;
+
+public class LanguageModelFFTest {
+
+ private static final float WEIGHT = 0.5f;
+
+ private LanguageModelFF ff;
+
+ @Before
+ public void setUp() {
+ Decoder.resetGlobalState();
+
+ FeatureVector weights = new FeatureVector();
+ weights.set("lm_0", WEIGHT);
+ String[] args = {"-lm_type", "berkeleylm", "-lm_order", "2", "-lm_file", "./joshua/test/lm/berkeley/lm"};
+
+ JoshuaConfiguration config = new JoshuaConfiguration();
+ ff = new LanguageModelFF(weights, args, config);
+ }
+
+ @After
+ public void tearDown() {
+ Decoder.resetGlobalState();
+ }
+
+ @Test
+ public void givenNonStartSymbol_whenEstimateFutureCost_thenMultipleWeightAndLogProbabilty() {
+ int[] left = {3};
+ NgramDPState currentState = new NgramDPState(left, new int[left.length]);
+
+ float score = ff.languageModel.sentenceLogProbability(left, 2, 1);
+ assertEquals(-99.0f, score, 0.0);
+
+ float cost = ff.estimateFutureCost(null, currentState, null);
+ assertEquals(score * WEIGHT, cost, 0.0);
+ }
+
+ @Test
+ public void givenOnlyStartSymbol_whenEstimateFutureCost_thenZeroResult() {
+ int startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ int[] left = {startSymbolId};
+ NgramDPState currentState = new NgramDPState(left, new int[left.length]);
+
+ float score = ff.languageModel.sentenceLogProbability(left, 2, 2);
+ assertEquals(0.0f, score, 0.0);
+
+ float cost = ff.estimateFutureCost(null, currentState, null);
+ assertEquals(score * WEIGHT, cost, 0.0);
+ }
+
+ @Test
+ public void givenStartAndOneMoreSymbol_whenEstimateFutureCost_thenMultipleWeightAndLogProbabilty() {
+ int startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ assertNotEquals(startSymbolId, 3);
+ int[] left = {startSymbolId, 3};
+ NgramDPState currentState = new NgramDPState(left, new int[left.length]);
+
+ float score = ff.languageModel.sentenceLogProbability(left, 2, 2);
+ assertEquals(-100.752754f, score, 0.0f);
+
+ float cost = ff.estimateFutureCost(null, currentState, null);
+ assertEquals(score * WEIGHT, cost, 0.0f);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/b68ccaae/tst/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java b/tst/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
new file mode 100644
index 0000000..6e0d90f
--- /dev/null
+++ b/tst/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
@@ -0,0 +1,62 @@
+package joshua.decoder.ff.lm.berkeley_lm;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import joshua.decoder.Decoder;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.Translation;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * Replacement for test/lm/berkeley/test.sh regression test
+ */
+@RunWith(Parameterized.class)
+public class LMGrammarBerkeleyTest {
+
+ private static final String INPUT = "the chat-rooms";
+ private static final String[] OPTIONS = "-v 0 -output-format %f".split(" ");
+
+ private JoshuaConfiguration joshuaConfig;
+ private Decoder decoder;
+
+ @Parameters
+ public static List<String> lmFiles() {
+ return Arrays.asList("resources/berkeley_lm/lm",
+ "resources/berkeley_lm/lm.gz",
+ "resources/berkeley_lm/lm.berkeleylm",
+ "resources/berkeley_lm/lm.berkeleylm.gz");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ }
+
+ @Parameter
+ public String lmFile;
+
+ @Test
+ public void verifyLM() {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.processCommandLineOptions(OPTIONS);
+ joshuaConfig.features.add("feature_function = LanguageModel -lm_type berkeleylm -lm_order 2 -lm_file " + lmFile);
+ decoder = new Decoder(joshuaConfig, null);
+ String translation = decode(INPUT).toString();
+ assertEquals(lmFile, "tm_glue_0=2.000 lm_0=-7.153\n", translation);
+ }
+
+ private Translation decode(String input) {
+ final Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+}
[08/10] incubator-joshua git commit: fixed recuser,
now triggered with separate -project-case flag
Posted by mj...@apache.org.
fixed recuser, now triggered with separate -project-case flag
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/73523c5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/73523c5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/73523c5b
Branch: refs/heads/master
Commit: 73523c5b67ff6de29718f897a70e648818809efc
Parents: 2fe4ad6
Author: Matt Post <po...@cs.jhu.edu>
Authored: Fri Apr 29 00:31:54 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Fri Apr 29 00:31:54 2016 -0400
----------------------------------------------------------------------
src/joshua/decoder/JoshuaConfiguration.java | 7 +++++++
test/decoder/lowercaser/output.gold | 2 ++
test/decoder/lowercaser/test.sh | 11 +++++++++--
3 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/73523c5b/src/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/JoshuaConfiguration.java b/src/joshua/decoder/JoshuaConfiguration.java
index 75a6162..a825268 100644
--- a/src/joshua/decoder/JoshuaConfiguration.java
+++ b/src/joshua/decoder/JoshuaConfiguration.java
@@ -55,6 +55,10 @@ public class JoshuaConfiguration {
// If set to true, Joshua will lowercase the input, creating an annotation that marks the
// original case
public boolean lowercase = false;
+
+ // If set to true, Joshua will recapitalize the output by projecting the case from aligned
+ // source-side words
+ public boolean project_case = false;
// List of grammar files to read
public ArrayList<String> tms = new ArrayList<String>();
@@ -650,6 +654,9 @@ public class JoshuaConfiguration {
} else if (parameter.equals(normalize_key("lowercase"))) {
lowercase = true;
+ } else if (parameter.equals(normalize_key("project-case"))) {
+ project_case = true;
+
} else {
if (parameter.equals(normalize_key("use-sent-specific-tm"))
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/73523c5b/test/decoder/lowercaser/output.gold
----------------------------------------------------------------------
diff --git a/test/decoder/lowercaser/output.gold b/test/decoder/lowercaser/output.gold
index 0c9c1eb..ea1d2bc 100644
--- a/test/decoder/lowercaser/output.gold
+++ b/test/decoder/lowercaser/output.gold
@@ -1,3 +1,5 @@
ELLA
she
+she
+She
SHE
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/73523c5b/test/decoder/lowercaser/test.sh
----------------------------------------------------------------------
diff --git a/test/decoder/lowercaser/test.sh b/test/decoder/lowercaser/test.sh
index 9f04e3a..875ae57 100755
--- a/test/decoder/lowercaser/test.sh
+++ b/test/decoder/lowercaser/test.sh
@@ -18,10 +18,17 @@
set -u
(
+# no match to phrase table, outputs ELLA
echo -e "ELLA" | $JOSHUA/bin/joshua-decoder -config config
-echo -e "Ella" | $JOSHUA/bin/joshua-decoder -config config -lowercase
+# matches phrase table, outputs she
echo -e "ELLA" | $JOSHUA/bin/joshua-decoder -config config -lowercase
-) > output 2> .log
+# matches phrase table, not capitalized because projected from first word of sentence, outputs she
+echo -e "Ella" | $JOSHUA/bin/joshua-decoder -config config -lowercase -project-case
+# matches phrase table, capitalized because of output-format
+echo -e "Ella" | $JOSHUA/bin/joshua-decoder -config config -lowercase -project-case -output-format %S
+# matches phrase table, projected case because all caps, outputs SHE
+echo -e "ELLA" | $JOSHUA/bin/joshua-decoder -config config -lowercase -project-case
+) > output 2> log
diff -u output output.gold > diff
[04/10] incubator-joshua git commit: Reworked most of the hypergraph
traversals for Viterbi and n-best extractions. Most importantly: translation
string extraction is now int based instead of doing regex matching and string
operations. This should be a l
Posted by mj...@apache.org.
Reworked most of the hypergraph traversals for Viterbi and n-best extractions. Most importantly: translation string extraction is now int based instead of doing regex matching and string operations. This should be a lot faster. However this will only work for hiero models for now. Phrase-based decoding will still use the String-based extractions. Before there were two ways to traverse the hypergraph: (1) regular tailNode order (used for Viterbi and WordAlignment) and (2) tailNode order according to target side non terminal indices (used for KBestExtraction). This caused quite some inconsistencies on how to write general extractors (output string, input string, feature vector, word alignments, tree, etc.) that support both. The main issue was that some extractors (String-based HypothesisExtractor) relies on the traversal order (2) to simply always merge children strings into the first nonTerminal symbol on the target side. However, this breaks the very same class when the input string
is requested (which is a supported feature in Joshua). This change gets rid of these inconsistencies for Hiero and simplifies a lot of the code. For phrase-based decoding, we still rely on the HypothesisExtractor. JoshuaConfiguration now throws an exception if you want to have the 'align_index' in output strings (Moses style) for Hiero models. This is not supported by int[]-based extraction and useless anyway.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/5b791285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/5b791285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/5b791285
Branch: refs/heads/master
Commit: 5b791285a95996279a51dd52b374c6363af78094
Parents: 3e23047
Author: Felix Hieber <fh...@amazon.com>
Authored: Wed Nov 25 15:11:27 2015 +0100
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Thu Apr 28 15:42:05 2016 -0700
----------------------------------------------------------------------
resources/wa_grammar | 6 +-
src/joshua/decoder/JoshuaConfiguration.java | 11 +-
src/joshua/decoder/StructuredTranslation.java | 72 ++----
src/joshua/decoder/Translation.java | 182 +++++--------
src/joshua/decoder/chart_parser/Chart.java | 8 +-
.../decoder/hypergraph/AlignedSourceTokens.java | 18 ++
.../decoder/hypergraph/AllSpansWalker.java | 4 +-
.../hypergraph/FeatureVectorExtractor.java | 80 ++++++
src/joshua/decoder/hypergraph/ForestWalker.java | 14 +-
.../GrammarBuilderWalkerFunction.java | 2 +-
src/joshua/decoder/hypergraph/HyperGraph.java | 10 +-
.../decoder/hypergraph/KBestExtractor.java | 254 +++++++------------
.../hypergraph/OutputStringExtractor.java | 195 ++++++++++++++
.../decoder/hypergraph/ViterbiExtractor.java | 159 ++++++------
.../ViterbiFeatureVectorWalkerFunction.java | 44 ----
.../ViterbiOutputStringWalkerFunction.java | 96 -------
.../decoder/hypergraph/WalkerFunction.java | 9 +-
.../hypergraph/WordAlignmentExtractor.java | 32 ++-
.../decoder/hypergraph/WordAlignmentState.java | 20 +-
src/joshua/decoder/io/JSONMessage.java | 2 +-
src/joshua/oracle/OracleExtractionHG.java | 6 +-
test/bn-en/hiero/input.bn | 2 +-
.../system/StructuredTranslationTest.java | 29 ++-
23 files changed, 677 insertions(+), 578 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/resources/wa_grammar
----------------------------------------------------------------------
diff --git a/resources/wa_grammar b/resources/wa_grammar
index 08c2c38..82d0052 100644
--- a/resources/wa_grammar
+++ b/resources/wa_grammar
@@ -1,3 +1,3 @@
-[X] ||| A [X,1] B1 [X,2] B2 C ||| a b [X,2] c1 [X,1] c2 ||| 1 1 1 1 1 1 ||| 0-0 2-1 4-1 5-3 5-5
-[X] ||| U Z1 Z2 ||| n1 u z ||| 1 1 1 1 1 1 ||| 0-1 1-2 2-2
-[X] ||| K ||| k1 k2 k3 n1 n2 n3 ||| 1 1 1 1 1 1 ||| 0-0 0-1 0-2
+[X] ||| A [X,1] B1 [X,2] B2 C ||| a b [X,2] c1 [X,1] c2 ||| 1 1 1 1 1 1 OOV=1 ||| 0-0 2-1 4-1 5-3 5-5
+[X] ||| U Z1 Z2 ||| n1 u z ||| 1 1 1 1 1 1 OOV=2 ||| 0-1 1-2 2-2
+[X] ||| K ||| k1 k2 k3 n1 n2 n3 ||| 1 1 1 1 1 1 OOV=4 ||| 0-0 0-1 0-2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/JoshuaConfiguration.java b/src/joshua/decoder/JoshuaConfiguration.java
index 6c8edf6..75a6162 100644
--- a/src/joshua/decoder/JoshuaConfiguration.java
+++ b/src/joshua/decoder/JoshuaConfiguration.java
@@ -598,9 +598,14 @@ public class JoshuaConfiguration {
search_algorithm = fds[1];
if (!search_algorithm.equals("cky") && !search_algorithm.equals("stack")) {
- System.err.println("* FATAL: -search must be one of 'stack' (for phrase-based decoding)");
- System.err.println("* or 'cky' (for hierarchical / syntactic decoding)");
- System.exit(1);
+ throw new RuntimeException(
+ "-search must be one of 'stack' (for phrase-based decoding) " +
+ "or 'cky' (for hierarchical / syntactic decoding)");
+ }
+
+ if (search_algorithm.equals("cky") && include_align_index) {
+ throw new RuntimeException(
+ "include_align_index is currently not supported with cky search");
}
} else if (parameter.equals(normalize_key("reordering-limit"))) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/StructuredTranslation.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/StructuredTranslation.java b/src/joshua/decoder/StructuredTranslation.java
index 1939ea0..7b2185f 100644
--- a/src/joshua/decoder/StructuredTranslation.java
+++ b/src/joshua/decoder/StructuredTranslation.java
@@ -1,20 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package joshua.decoder;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static joshua.decoder.hypergraph.ViterbiExtractor.walk;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignmentList;
+import static joshua.util.FormatUtils.removeSentenceMarkers;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import joshua.decoder.ff.FeatureFunction;
import joshua.decoder.hypergraph.HyperGraph;
-import joshua.decoder.hypergraph.ViterbiFeatureVectorWalkerFunction;
-import joshua.decoder.hypergraph.ViterbiOutputStringWalkerFunction;
-import joshua.decoder.hypergraph.WalkerFunction;
-import joshua.decoder.hypergraph.WordAlignmentExtractor;
import joshua.decoder.segment_file.Sentence;
/**
@@ -30,13 +45,11 @@ import joshua.decoder.segment_file.Sentence;
public class StructuredTranslation {
private final Sentence sourceSentence;
- private final List<FeatureFunction> featureFunctions;
-
private final String translationString;
private final List<String> translationTokens;
private final float translationScore;
- private List<List<Integer>> translationWordAlignments;
- private Map<String,Float> translationFeatures;
+ private final List<List<Integer>> translationWordAlignments;
+ private final Map<String,Float> translationFeatures;
private final float extractionTime;
public StructuredTranslation(final Sentence sourceSentence,
@@ -46,35 +59,14 @@ public class StructuredTranslation {
final long startTime = System.currentTimeMillis();
this.sourceSentence = sourceSentence;
- this.featureFunctions = featureFunctions;
- this.translationString = extractViterbiString(hypergraph);
+ this.translationString = removeSentenceMarkers(getViterbiString(hypergraph));
this.translationTokens = extractTranslationTokens();
this.translationScore = extractTranslationScore(hypergraph);
- this.translationFeatures = extractViterbiFeatures(hypergraph);
- this.translationWordAlignments = extractViterbiWordAlignment(hypergraph);
+ this.translationFeatures = getViterbiFeatures(hypergraph, featureFunctions, sourceSentence).getMap();
+ this.translationWordAlignments = getViterbiWordAlignmentList(hypergraph);
this.extractionTime = (System.currentTimeMillis() - startTime) / 1000.0f;
}
- private Map<String,Float> extractViterbiFeatures(final HyperGraph hypergraph) {
- if (hypergraph == null) {
- return emptyMap();
- } else {
- ViterbiFeatureVectorWalkerFunction viterbiFeatureVectorWalker = new ViterbiFeatureVectorWalkerFunction(featureFunctions, sourceSentence);
- walk(hypergraph.goalNode, viterbiFeatureVectorWalker);
- return new HashMap<String,Float>(viterbiFeatureVectorWalker.getFeaturesMap());
- }
- }
-
- private List<List<Integer>> extractViterbiWordAlignment(final HyperGraph hypergraph) {
- if (hypergraph == null) {
- return emptyList();
- } else {
- final WordAlignmentExtractor wordAlignmentWalker = new WordAlignmentExtractor();
- walk(hypergraph.goalNode, wordAlignmentWalker);
- return wordAlignmentWalker.getFinalWordAlignments();
- }
- }
-
private float extractTranslationScore(final HyperGraph hypergraph) {
if (hypergraph == null) {
return 0;
@@ -83,16 +75,6 @@ public class StructuredTranslation {
}
}
- private String extractViterbiString(final HyperGraph hypergraph) {
- if (hypergraph == null) {
- return sourceSentence.source();
- } else {
- final WalkerFunction viterbiOutputStringWalker = new ViterbiOutputStringWalkerFunction();
- walk(hypergraph.goalNode, viterbiOutputStringWalker);
- return viterbiOutputStringWalker.toString();
- }
- }
-
private List<String> extractTranslationTokens() {
if (translationString.isEmpty()) {
return emptyList();
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/Translation.java b/src/joshua/decoder/Translation.java
index 85f4974..8004d9f 100644
--- a/src/joshua/decoder/Translation.java
+++ b/src/joshua/decoder/Translation.java
@@ -18,18 +18,21 @@
*/
package joshua.decoder;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignments;
+import static joshua.util.FormatUtils.removeSentenceMarkers;
+
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.StringWriter;
-import java.util.Arrays;
import java.util.List;
import joshua.decoder.ff.FeatureFunction;
+import joshua.decoder.ff.FeatureVector;
import joshua.decoder.ff.lm.StateMinimizingLanguageModel;
import joshua.decoder.hypergraph.HyperGraph;
import joshua.decoder.hypergraph.KBestExtractor;
-import joshua.decoder.hypergraph.ViterbiExtractor;
-import joshua.decoder.hypergraph.WordAlignmentState;
import joshua.decoder.io.DeNormalize;
import joshua.decoder.segment_file.Sentence;
@@ -50,18 +53,7 @@ public class Translation {
*/
private String output = null;
- /**
- * The raw one-best translation.
- */
- private String rawTranslation = null;
-
- public String rawTranslation() {
- return rawTranslation;
- }
-
- private WordAlignmentState alignment = null;
- private float score = 0;
- private float translationTime;
+ private StructuredTranslation structuredTranslation = null;
public Translation(Sentence source, HyperGraph hypergraph,
List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) {
@@ -69,8 +61,9 @@ public class Translation {
if (joshuaConfiguration.use_structured_output) {
- // create structured output instead of the String manipulation below.
- createStructuredOutput(source, hypergraph);
+ structuredTranslation = new StructuredTranslation(
+ source, hypergraph, featureFunctions);
+ this.output = structuredTranslation.getTranslationString();
} else {
@@ -85,54 +78,59 @@ public class Translation {
long startTime = System.currentTimeMillis();
- // We must put this weight as zero, otherwise we get an error when we try to retrieve it
- // without checking
- Decoder.weights.increment("BLEU", 0);
-
- rawTranslation = ViterbiExtractor.extractViterbiString(hypergraph.goalNode).trim();
- rawTranslation = rawTranslation.substring(new String("<s>").length() + 1, rawTranslation.lastIndexOf("</s>"));
-
- Decoder.LOG(1, String.format("Translation %d: %.3f %s", source.id(), hypergraph.goalNode.getScore(),
- rawTranslation));
-
- if (joshuaConfiguration.topN == 0) {
-
- /*
- * Setting topN to 0 turns off k-best extraction, in which case we need to parse through
- * the output-string, with the understanding that we can only substitute variables for the
- * output string, sentence number, and model score.
- */
- String translation = joshuaConfiguration.outputFormat.replace("%s", rawTranslation)
- .replace("%S", DeNormalize.processSingleLine(rawTranslation))
- .replace("%c", String.format("%.3f", hypergraph.goalNode.getScore()))
- .replace("%i", String.format("%d", source.id()));
-
- /* %a causes output of word level alignments between input and output hypothesis */
- if (joshuaConfiguration.outputFormat.contains("%a")) {
- translation = translation.replace("%a", ViterbiExtractor.extractViterbiAlignment(hypergraph.goalNode));
- }
-
- out.write(translation);
- out.newLine();
+ // We must put this weight as zero, otherwise we get an error when we try to retrieve it
+ // without checking
+ Decoder.weights.increment("BLEU", 0);
- } else {
- KBestExtractor kBestExtractor = new KBestExtractor(source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
- kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
-
- if (joshuaConfiguration.rescoreForest) {
- Decoder.weights.increment("BLEU", joshuaConfiguration.rescoreForestWeight);
+ if (joshuaConfiguration.topN == 0) {
+
+ /* construct Viterbi output */
+ final String best = getViterbiString(hypergraph);
+
+ Decoder.LOG(1, String.format("Translation %d: %.3f %s", source.id(), hypergraph.goalNode.getScore(),
+ best));
+
+ /*
+ * Setting topN to 0 turns off k-best extraction, in which case we need to parse through
+ * the output-string, with the understanding that we can only substitute variables for the
+ * output string, sentence number, and model score.
+ */
+ String translation = joshuaConfiguration.outputFormat
+ .replace("%s", removeSentenceMarkers(best))
+ .replace("%S", DeNormalize.processSingleLine(best))
+ .replace("%c", String.format("%.3f", hypergraph.goalNode.getScore()))
+ .replace("%i", String.format("%d", source.id()));
+
+ if (joshuaConfiguration.outputFormat.contains("%a")) {
+ translation = translation.replace("%a", getViterbiWordAlignments(hypergraph));
+ }
+
+ if (joshuaConfiguration.outputFormat.contains("%f")) {
+ final FeatureVector features = getViterbiFeatures(hypergraph, featureFunctions, source);
+ translation = translation.replace("%f", joshuaConfiguration.moses ? features.mosesString() : features.toString());
+ }
+
+ out.write(translation);
+ out.newLine();
+
+ } else {
+
+ final KBestExtractor kBestExtractor = new KBestExtractor(
+ source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
- Decoder.weights.increment("BLEU", -joshuaConfiguration.rescoreForestWeight);
- kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+ if (joshuaConfiguration.rescoreForest) {
+ Decoder.weights.increment("BLEU", joshuaConfiguration.rescoreForestWeight);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+
+ Decoder.weights.increment("BLEU", -joshuaConfiguration.rescoreForestWeight);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+ }
}
- }
float seconds = (float) (System.currentTimeMillis() - startTime) / 1000.0f;
Decoder.LOG(1, String.format("Input %d: %d-best extraction took %.3f seconds", id(),
joshuaConfiguration.topN, seconds));
- this.translationTime = seconds;
-
} else {
@@ -175,39 +173,6 @@ public class Translation {
}
- /**
- * Instead of returning a single string with output information appended
- * (if JoshuaConfig.use_structured_output == false),
- * write Viterbi information (score, translation, word alignment) to member
- * variables for easier access from outside pipelines.
- */
- private void createStructuredOutput(Sentence source, HyperGraph hypergraph) {
-
- this.translationTime = 0;
-
- long startTime = System.currentTimeMillis();
-
- if (hypergraph != null) {
-
- this.output = ViterbiExtractor.extractViterbiString(hypergraph.goalNode).trim();
- // trims whitespaces (same idiom as in existing Joshua code (65)
- this.output = this.output.substring(this.output.indexOf(' ') + 1, this.output.lastIndexOf(' '));
- this.alignment = ViterbiExtractor.buildViterbiAlignment(hypergraph.goalNode);
- this.score = hypergraph.goalNode.getScore();
-
- } else {
-
- this.output = this.source.source();
- this.alignment = null;
-
- }
-
- this.translationTime = (System.currentTimeMillis() - startTime) / 1000.0f;
-
- Decoder.LOG(1, String.format("Translation %d: %.3f %s (%.3f)", source.id(), hypergraph.goalNode.getScore(), this.output, this.translationTime));
-
- }
-
public Sentence getSourceSentence() {
return this.source;
}
@@ -221,32 +186,17 @@ public class Translation {
return output;
}
- public float getTranslationTime() {
- return translationTime;
- }
-
- public String getTranslationString() {
- return (output == null) ? "" : output.trim();
- }
-
- public List<List<Integer>> getWordAlignment() {
- return alignment.toFinalList();
- }
-
- public String getWordAlignmentString() {
- return (alignment == null) ? "" : alignment.toFinalString();
- }
-
- public float getTranslationScore() {
- return score;
- }
-
- public List<String> getTranslationTokens() {
- return Arrays.asList(getTranslationString().split("\\s+"));
- }
-
- public String getDeNormalizedTranslation() {
- return DeNormalize.processSingleLine(getTranslationString());
+ /**
+ * Returns the StructuredTranslation object
+ * if JoshuaConfiguration.construct_structured_output == True.
+ * @throws RuntimeException if StructuredTranslation object not set.
+ * @return
+ */
+ public StructuredTranslation getStructuredTranslation() {
+ if (structuredTranslation == null) {
+ throw new RuntimeException("No StructuredTranslation object created. You should set JoshuaConfigration.construct_structured_output = true");
+ }
+ return structuredTranslation;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/chart_parser/Chart.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/chart_parser/Chart.java b/src/joshua/decoder/chart_parser/Chart.java
index c400f08..b10c013 100644
--- a/src/joshua/decoder/chart_parser/Chart.java
+++ b/src/joshua/decoder/chart_parser/Chart.java
@@ -118,8 +118,8 @@ public class Chart {
*/
public Chart(Sentence sentence, List<FeatureFunction> featureFunctions, Grammar[] grammars,
- String goalSymbol, JoshuaConfiguration config2) {
- this.config = config2;
+ String goalSymbol, JoshuaConfiguration config) {
+ this.config = config;
this.inputLattice = sentence.getLattice();
this.sourceLength = inputLattice.size() - 1;
this.featureFunctions = featureFunctions;
@@ -142,9 +142,9 @@ public class Chart {
for (int i = 0; i < grammars.length; i++)
this.grammars[i + 1] = grammars[i];
- MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", config2);
+ MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", this.config);
AbstractGrammar.addOOVRules(oovGrammar, sentence.getLattice(), featureFunctions,
- config.true_oovs_only);
+ this.config.true_oovs_only);
this.grammars[0] = oovGrammar;
// each grammar will have a dot chart
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/AlignedSourceTokens.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/AlignedSourceTokens.java b/src/joshua/decoder/hypergraph/AlignedSourceTokens.java
index eb92f0d..5c6b2dd 100644
--- a/src/joshua/decoder/hypergraph/AlignedSourceTokens.java
+++ b/src/joshua/decoder/hypergraph/AlignedSourceTokens.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package joshua.decoder.hypergraph;
import java.util.LinkedList;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/AllSpansWalker.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/AllSpansWalker.java b/src/joshua/decoder/hypergraph/AllSpansWalker.java
index 9f2ec36..3964bb2 100644
--- a/src/joshua/decoder/hypergraph/AllSpansWalker.java
+++ b/src/joshua/decoder/hypergraph/AllSpansWalker.java
@@ -48,11 +48,11 @@ public class AllSpansWalker {
public void walk(HGNode node, final WalkerFunction walker) {
new ForestWalker().walk(node, new joshua.decoder.hypergraph.WalkerFunction() {
@Override
- public void apply(HGNode node) {
+ public void apply(HGNode node, int index) {
if (node != null) {
Span span = new Span(node.i, node.j);
if (!visitedSpans.contains(span)) {
- walker.apply(node);
+ walker.apply(node, 0);
visitedSpans.add(span);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/FeatureVectorExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/FeatureVectorExtractor.java b/src/joshua/decoder/hypergraph/FeatureVectorExtractor.java
new file mode 100644
index 0000000..dbe4f4b
--- /dev/null
+++ b/src/joshua/decoder/hypergraph/FeatureVectorExtractor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.hypergraph;
+
+import static joshua.decoder.chart_parser.ComputeNodeResult.computeTransitionFeatures;
+
+import java.util.List;
+
+import joshua.decoder.ff.FeatureFunction;
+import joshua.decoder.ff.FeatureVector;
+import joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * During decoding, individual features values are not stored, only the model score on each edge.
+ * This saves space. If you want to print the actual feature values, they have to be assembled
+ * from the edges of the derivation, which means replaying the feature functions. This visitor
+ * does just that, using the generic derivation visitor.
+ */
+public class FeatureVectorExtractor implements WalkerFunction, DerivationVisitor {
+
+ private final FeatureVector features;
+ private final List<FeatureFunction> featureFunctions;
+ private final Sentence sourceSentence;
+
+ public FeatureVectorExtractor(
+ final List<FeatureFunction> featureFunctions,
+ final Sentence sourceSentence) {
+ this.features = new FeatureVector();
+ this.featureFunctions = featureFunctions;
+ this.sourceSentence = sourceSentence;
+ }
+
+ /** Accumulate edge features from Viterbi path */
+ @Override
+ public void apply(HGNode node, int nodeIndex) {
+ features.add(
+ computeTransitionFeatures(
+ featureFunctions,
+ node.bestHyperedge,
+ node.i, node.j,
+ sourceSentence));
+ }
+
+ /** Accumulate edge features for that DerivationState */
+ @Override
+ public void before(DerivationState state, int level, int tailNodeIndex) {
+ features.add(
+ computeTransitionFeatures(
+ featureFunctions,
+ state.edge,
+ state.parentNode.i, state.parentNode.j,
+ sourceSentence));
+ }
+
+ /** Nothing to do */
+ @Override
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
+
+ public FeatureVector getFeatures() {
+ return features;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/ForestWalker.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/ForestWalker.java b/src/joshua/decoder/hypergraph/ForestWalker.java
index bc0ee21..1838082 100644
--- a/src/joshua/decoder/hypergraph/ForestWalker.java
+++ b/src/joshua/decoder/hypergraph/ForestWalker.java
@@ -46,8 +46,12 @@ public class ForestWalker {
this.traversalType = traversal;
visitedNodes = new HashSet<HGNode>();
}
-
+
public void walk(HGNode node, WalkerFunction walker) {
+ walk(node, walker, 0);
+ }
+
+ public void walk(HGNode node, WalkerFunction walker, int nodeIndex) {
// short circuit
if (visitedNodes.contains(node))
return;
@@ -55,19 +59,21 @@ public class ForestWalker {
visitedNodes.add(node);
if (this.traversalType == TRAVERSAL.PREORDER)
- walker.apply(node);
+ walker.apply(node, 0);
if (node.getHyperEdges() != null) {
for (HyperEdge edge : node.getHyperEdges()) {
if (edge.getTailNodes() != null) {
+ int tailNodeIndex = 0;
for (HGNode tailNode : edge.getTailNodes()) {
- walk(tailNode, walker);
+ walk(tailNode, walker, tailNodeIndex);
+ tailNodeIndex++;
}
}
}
}
if (this.traversalType == TRAVERSAL.POSTORDER)
- walker.apply(node);
+ walker.apply(node, nodeIndex);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java b/src/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
index d3d57d1..5a3b422 100644
--- a/src/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
+++ b/src/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
@@ -61,7 +61,7 @@ public class GrammarBuilderWalkerFunction implements WalkerFunction {
outStream = out;
}
- public void apply(HGNode node) {
+ public void apply(HGNode node, int index) {
// System.err.printf("VISITING NODE: %s\n", getLabelWithSpan(node));
for (HyperEdge e : node.hyperedges) {
Rule r = getRuleWithSpans(e, node);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/HyperGraph.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/HyperGraph.java b/src/joshua/decoder/hypergraph/HyperGraph.java
index 4443270..2d1cfb0 100644
--- a/src/joshua/decoder/hypergraph/HyperGraph.java
+++ b/src/joshua/decoder/hypergraph/HyperGraph.java
@@ -84,7 +84,7 @@ public class HyperGraph {
}
@Override
- public void apply(HGNode node) {
+ public void apply(HGNode node, int index) {
if (! nodesVisited.contains(node)) {
if (node.bestHyperedge.getRule() != null) {
hg.numNodes++;
@@ -98,21 +98,19 @@ public class HyperGraph {
private class HyperGraphDumper implements WalkerFunction {
private int node_number = 1;
- private int sentID = -1;
private List<FeatureFunction> model = null;
private PrintWriter out = null;
private HashMap<HGNode, Integer> nodeMap;
- public HyperGraphDumper(PrintWriter out, int sentID, List<FeatureFunction> model) {
+ public HyperGraphDumper(PrintWriter out, List<FeatureFunction> model) {
this.out = out;
- this.sentID = sentID;
this.model = model;
this.nodeMap = new HashMap<HGNode, Integer>();
}
@Override
- public void apply(HGNode node) {
+ public void apply(HGNode node, int index) {
if (! nodeMap.containsKey(node)) { // Make sure each node is listed only once
nodeMap.put(node, this.node_number);
@@ -157,7 +155,7 @@ public class HyperGraph {
count();
out.println("# target ||| features");
out.println(String.format("%d %d", numNodes, numEdges));
- new ForestWalker(TRAVERSAL.POSTORDER).walk(this.goalNode, new HyperGraphDumper(out, sentence.id(), model));
+ new ForestWalker(TRAVERSAL.POSTORDER).walk(this.goalNode, new HyperGraphDumper(out, model));
out.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/KBestExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/KBestExtractor.java b/src/joshua/decoder/hypergraph/KBestExtractor.java
index 45b9ccb..9107cea 100644
--- a/src/joshua/decoder/hypergraph/KBestExtractor.java
+++ b/src/joshua/decoder/hypergraph/KBestExtractor.java
@@ -18,6 +18,9 @@
*/
package joshua.decoder.hypergraph;
+import static joshua.util.FormatUtils.unescapeSpecialSymbols;
+import static joshua.util.FormatUtils.removeSentenceMarkers;
+
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
@@ -34,7 +37,6 @@ import java.util.regex.Matcher;
import joshua.corpus.Vocabulary;
import joshua.decoder.BLEU;
import joshua.decoder.JoshuaConfiguration;
-import joshua.decoder.chart_parser.ComputeNodeResult;
import joshua.decoder.ff.FeatureFunction;
import joshua.decoder.ff.FeatureVector;
import joshua.decoder.ff.fragmentlm.Tree;
@@ -92,6 +94,8 @@ import joshua.util.FormatUtils;
*/
public class KBestExtractor {
private final JoshuaConfiguration joshuaConfiguration;
+ private final String outputFormat;
+ private final boolean stackDecoding;
private final HashMap<HGNode, VirtualNode> virtualNodesTable = new HashMap<HGNode, VirtualNode>();
// static final String rootSym = JoshuaConfiguration.goal_symbol;
@@ -103,35 +107,36 @@ public class KBestExtractor {
};
/* Whether to extract only unique strings */
- private boolean extractUniqueNbest = true;
-
- /* Whether to include the alignment information in the output */
- @SuppressWarnings("unused")
- private boolean includeAlign = false;
+ private final boolean extractUniqueNbest;
/* Which side to output (source or target) */
- private Side defaultSide = Side.TARGET;
+ private final Side defaultSide;
/* The input sentence */
- private Sentence sentence;
+ private final Sentence sentence;
/* The weights being used to score the forest */
- private FeatureVector weights;
+ private final FeatureVector weights;
/* The feature functions */
- private List<FeatureFunction> models;
+ private final List<FeatureFunction> featureFunctions;
/* BLEU statistics of the references */
- BLEU.References references = null;
+ private BLEU.References references = null;
- public KBestExtractor(Sentence sentence, List<FeatureFunction> models, FeatureVector weights,
- boolean isMonolingual, JoshuaConfiguration joshuaConfiguration) {
+ public KBestExtractor(
+ Sentence sentence,
+ List<FeatureFunction> featureFunctions,
+ FeatureVector weights,
+ boolean isMonolingual,
+ JoshuaConfiguration joshuaConfiguration) {
- this.models = models;
+ this.featureFunctions = featureFunctions;
this.joshuaConfiguration = joshuaConfiguration;
+ this.outputFormat = this.joshuaConfiguration.outputFormat;
+ this.stackDecoding = this.joshuaConfiguration.search_algorithm.equals("stack");
this.extractUniqueNbest = joshuaConfiguration.use_unique_nbest;
- this.includeAlign = joshuaConfiguration.include_align_index;
this.weights = weights;
this.defaultSide = (isMonolingual ? Side.SOURCE : Side.TARGET);
@@ -172,45 +177,44 @@ public class KBestExtractor {
// DerivationState derivationState = getKthDerivation(node, k);
if (derivationState != null) {
// ==== read the kbest from each hgnode and convert to output format
- FeatureVector features = new FeatureVector();
-
+ String hypothesis = unescapeSpecialSymbols(
+ removeSentenceMarkers(
+ derivationState.getHypothesis()));
+
/*
- * To save space, the decoder only stores the model cost, no the individual feature values. If
- * you want to output them, you have to replay them.
+ * To save space, the decoder only stores the model cost,
+ * no the individual feature values.
+ * If you want to output them, you have to replay them.
*/
- String hypothesis = null;
- if (joshuaConfiguration.outputFormat.contains("%f")
- || joshuaConfiguration.outputFormat.contains("%d"))
- features = derivationState.replayFeatures();
-
- hypothesis = derivationState.getHypothesis()
- .replaceAll("-lsb-", "[")
- .replaceAll("-rsb-", "]")
- .replaceAll("-pipe-", "|");
-
- outputString = joshuaConfiguration.outputFormat
+ FeatureVector features = new FeatureVector();
+ if (outputFormat.contains("%f") || outputFormat.contains("%d"))
+ features = derivationState.getFeatures();
+
+ outputString = outputFormat
.replace("%k", Integer.toString(k))
- .replace("%s", recapitalize(hypothesis, node))
- .replace("%S", DeNormalize.processSingleLine(recapitalize(hypothesis, node)))
+ .replace("%s", hypothesis)
+ .replace("%S", DeNormalize.processSingleLine(hypothesis))
+ // TODO (kellens): Fix the recapitalization here
.replace("%i", Integer.toString(sentence.id()))
.replace("%f", joshuaConfiguration.moses ? features.mosesString() : features.toString())
.replace("%c", String.format("%.3f", derivationState.cost));
- if (joshuaConfiguration.outputFormat.contains("%t")) {
+ if (outputFormat.contains("%t")) {
outputString = outputString.replace("%t", derivationState.getTree());
}
- if (joshuaConfiguration.outputFormat.contains("%e"))
- outputString = outputString.replace("%e", derivationState.getHypothesis(Side.SOURCE));
+ if (outputFormat.contains("%e")) {
+ outputString = outputString.replace("%e", removeSentenceMarkers(derivationState.getHypothesis(Side.SOURCE)));
+ }
/* %d causes a derivation with rules one per line to be output */
- if (joshuaConfiguration.outputFormat.contains("%d")) {
+ if (outputFormat.contains("%d")) {
outputString = outputString.replace("%d", derivationState.getDerivation());
}
/* %a causes output of word level alignments between input and output hypothesis */
- if (joshuaConfiguration.outputFormat.contains("%a")) {
+ if (outputFormat.contains("%a")) {
outputString = outputString.replace("%a", derivationState.getWordAlignment());
}
@@ -285,36 +289,6 @@ public class KBestExtractor {
}
return virtualNode;
}
-
- private String recapitalize(String input, HGNode goalNode) {
- WordAlignmentState alignment = ViterbiExtractor.buildViterbiAlignment(goalNode);
-
- String[] tokens = input.split("\\s+");
-
- List<List<Integer>> points = alignment.toFinalList();
- for (int i = 0; i < points.size(); i++) {
- List<Integer> target = points.get(i);
- for (int source: target) {
- Token token = sentence.getTokens().get(source + 1); // skip <s>
- String annotation = "";
- if (token != null && token.getAnnotation("lettercase") != null)
- annotation = token.getAnnotation("lettercase");
- if (source != 0 && annotation.equals("upper"))
- tokens[i] = FormatUtils.capitalize(tokens[i]);
- else if (annotation.equals("all-upper"))
- tokens[i] = tokens[i].toUpperCase();
- }
- }
-
- String cap = new String();
- for (int i = 0; i < tokens.length; i++) {
- if (i > 0)
- cap += " ";
- cap += tokens[i];
- }
- return cap;
- }
-
/**
* This class is essentially a wrapper around an HGNode, annotating it with information needed to
@@ -384,7 +358,8 @@ public class KBestExtractor {
if (extractUniqueNbest) {
// We pass false for extract_nbest_tree because we want; to check that the hypothesis
// *strings* are unique, not the trees.
- String res_str = derivationState.getHypothesis();
+ final String res_str = derivationState.getHypothesis();
+
if (!uniqueStringsTable.contains(res_str)) {
nbests.add(derivationState);
uniqueStringsTable.add(res_str);
@@ -636,10 +611,6 @@ public class KBestExtractor {
bleu = 0.0f;
}
- public String getWordAlignment() {
- return visit(new WordAlignmentExtractor()).toString();
- }
-
/**
* Computes a scaled approximate BLEU from the accumulated statistics. We know the number of
* words; to compute the effective reference length, we take the real reference length statistic
@@ -736,50 +707,61 @@ public class KBestExtractor {
* Visits every state in the derivation in a depth-first order.
*/
private DerivationVisitor visit(DerivationVisitor visitor) {
- return visit(visitor, 0);
+ return visit(visitor, 0, 0);
}
- private DerivationVisitor visit(DerivationVisitor visitor, int indent) {
+ private DerivationVisitor visit(DerivationVisitor visitor, int indent, int tailNodeIndex) {
- visitor.before(this, indent);
+ visitor.before(this, indent, tailNodeIndex);
- Rule rule = edge.getRule();
+ final Rule rule = edge.getRule();
+ final List<HGNode> tailNodes = edge.getTailNodes();
- if (null == rule) {
- getChildDerivationState(edge, 0).visit(visitor, indent + 1);
+ if (rule == null) {
+ getChildDerivationState(edge, 0).visit(visitor, indent + 1, 0);
} else {
- if (edge.getTailNodes() != null) {
- int[] english = rule.getEnglish();
- for (int c = 0; c < english.length; c++) {
- if (Vocabulary.nt(english[c])) {
- int index = -(english[c] + 1);
- getChildDerivationState(edge, index).visit(visitor, indent + 1);
- }
+ if (tailNodes != null) {
+ for (int index = 0; index < tailNodes.size(); index++) {
+ getChildDerivationState(edge, index).visit(visitor, indent + 1, index);
}
}
}
- visitor.after(this, indent);
+ visitor.after(this, indent, tailNodeIndex);
return visitor;
}
- private String getHypothesis() {
- return getHypothesis(defaultSide);
+ private String getWordAlignment() {
+ return visit(new WordAlignmentExtractor()).toString();
}
private String getTree() {
return visit(new TreeExtractor()).toString();
}
+
+ private String getHypothesis() {
+ return getHypothesis(defaultSide);
+ }
- private String getHypothesis(Side side) {
- return visit(new HypothesisExtractor(side)).toString();
+ /**
+ * For stack decoding we keep using the old string-based
+ * HypothesisExtractor.
+ * For Hiero, we use a faster, int-based hypothesis extraction
+ * that is correct also for Side.SOURCE cases.
+ */
+ private String getHypothesis(final Side side) {
+ if (stackDecoding) {
+ return visit(new HypothesisExtractor(side)).toString();
+ } else {
+ return visit(new OutputStringExtractor(side.equals(Side.SOURCE))).toString();
+ }
}
- private FeatureVector replayFeatures() {
- FeatureReplayer fp = new FeatureReplayer();
- visit(fp);
- return fp.getFeatures();
+ private FeatureVector getFeatures() {
+ final FeatureVectorExtractor extractor = new FeatureVectorExtractor(featureFunctions, sentence);
+ visit(extractor);
+ return extractor.getFeatures();
}
private String getDerivation() {
@@ -831,25 +813,26 @@ public class KBestExtractor {
*
* @param state the derivation state
* @param level the tree depth
+ * @param tailNodeIndex the tailNodeIndex corresponding to state
*/
- void before(DerivationState state, int level);
+ void before(DerivationState state, int level, int tailNodeIndex);
/**
* Called after a node's children have been visited.
*
* @param state the derivation state
* @param level the tree depth
+ * @param tailNodeIndex the tailNodeIndex corresponding to state
*/
- void after(DerivationState state, int level);
+ void after(DerivationState state, int level, int tailNodeIndex);
}
-
+
/**
- * Extracts the hypothesis from the leaves of the tree using the generic (depth-first) visitor.
- * Since we're using the visitor, we can't just print out the words as we see them. We have to
- * print the words to the left of the nonterminal, then recursively print that nonterminal, then
- * the words after it, and so on. To accomplish this, we add rules to a stack, merging the words
- * from terminal productions into the most recent nonterminal in the stack.
- *
+ * String-based HypothesisExtractor that works with Joshua's Phrase decoder.
+ * It is slower than an int[]-based extraction due to many String operations.
+ * It also contains a bug for Hiero KBest with NT-reordering rules when
+ * the source side is extracted (due to its dependency on target-side ordered
+ * hypergraph traversal.
*/
public class HypothesisExtractor implements DerivationVisitor {
@@ -879,7 +862,7 @@ public class KBestExtractor {
* Whenever we reach a rule in the depth-first derivaiton, we add it to the stack
* via a call to the merge() function.
*/
- public void before(DerivationState state, int level) {
+ public void before(DerivationState state, int level, int tailNodeIndex) {
Rule rule = state.edge.getRule();
if (rule != null)
if (side == Side.TARGET) {
@@ -898,7 +881,7 @@ public class KBestExtractor {
}
@Override
- public void after(DerivationState state, int level) {
+ public void after(DerivationState state, int level, int tailNodeIndex) {
}
/**
@@ -927,7 +910,7 @@ public class KBestExtractor {
* and merge it into the tree we're building.
*/
@Override
- public void before(DerivationState state, int indent) {
+ public void before(DerivationState state, int indent, int tailNodeIndex) {
HyperEdge edge = state.edge;
Rule rule = edge.getRule();
@@ -972,7 +955,7 @@ public class KBestExtractor {
}
@Override
- public void after(DerivationState state, int indent) {
+ public void after(DerivationState state, int indent, int tailNodeIndex) {
// do nothing
}
@@ -1012,7 +995,7 @@ public class KBestExtractor {
}
@Override
- public void before(DerivationState state, int indent) {
+ public void before(DerivationState state, int indent, int tailNodeIndex) {
HyperEdge edge = state.edge;
Rule rule = edge.getRule();
@@ -1022,9 +1005,9 @@ public class KBestExtractor {
for (int i = 0; i < indent * 2; i++)
sb.append(" ");
- FeatureReplayer replayer = new FeatureReplayer();
- replayer.before(state, indent);
- FeatureVector transitionFeatures = replayer.getFeatures();
+ final FeatureVectorExtractor extractor = new FeatureVectorExtractor(featureFunctions, sentence);
+ extractor.before(state, indent, tailNodeIndex);
+ final FeatureVector transitionFeatures = extractor.getFeatures();
// sb.append(rule).append(" ||| " + features + " ||| " +
// KBestExtractor.this.weights.innerProduct(features));
@@ -1048,53 +1031,8 @@ public class KBestExtractor {
}
@Override
- public void after(DerivationState state, int level) {
- // TODO Auto-generated method stub
- }
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
}
+
- /**
- * During decoding, individual features values are not stored, only the model score on each edge.
- * This saves space. If you want to print the actual feature values, they have to be assembled
- * from the edges of the derivation, which means replaying the feature functions. This visitor
- * does just that, using the generic derivation visitor.
- */
- public class FeatureReplayer implements DerivationVisitor {
-
- private FeatureVector features;
-
- public FeatureReplayer() {
- features = new FeatureVector();
- }
-
- public FeatureReplayer(FeatureVector useThese) {
- features = useThese;
- }
-
- public FeatureVector getFeatures() {
- return features;
- }
-
- /**
- * We could do this in either before() or after().
- */
- @Override
- public void before(DerivationState state, int level) {
- if (features != null) {
- HGNode parentNode = state.parentNode;
-
- HyperEdge edge = state.edge;
-
- FeatureVector transitionCosts = ComputeNodeResult.computeTransitionFeatures(models, edge,
- parentNode.i, parentNode.j, sentence);
-
- features.add(transitionCosts);
- }
- }
-
- @Override
- public void after(DerivationState state, int level) {
- // Nothing to do
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/OutputStringExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/OutputStringExtractor.java b/src/joshua/decoder/hypergraph/OutputStringExtractor.java
new file mode 100644
index 0000000..acb2e17
--- /dev/null
+++ b/src/joshua/decoder/hypergraph/OutputStringExtractor.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.hypergraph;
+
+import static java.lang.Math.min;
+import static joshua.corpus.Vocabulary.getWords;
+import static joshua.corpus.Vocabulary.nt;
+
+import java.util.Stack;
+
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor;
+
+public class OutputStringExtractor implements WalkerFunction, DerivationVisitor {
+
+ public OutputStringExtractor(final boolean extractSource) {
+ this.extractSource = extractSource;
+ }
+
+ private Stack<OutputString> outputStringStack = new Stack<>();
+ private final boolean extractSource;
+
+ @Override
+ public void apply(HGNode node, int nodeIndex) {
+ apply(node.bestHyperedge.getRule(), nodeIndex);
+ }
+
+ /**
+ * Visiting a node during k-best extraction is the same as
+ * apply() for Viterbi extraction but using the edge from
+ * the Derivation state.
+ */
+ @Override
+ public void before(final DerivationState state, int level, int tailNodeIndex) {
+ apply(state.edge.getRule(), tailNodeIndex);
+ }
+
+ private void apply(Rule rule, int nodeIndex) {
+ if (rule != null) {
+ final int[] words = extractSource ? rule.getFrench() : rule.getEnglish();
+ merge(new OutputString(words, rule.getArity(), nodeIndex));
+ }
+ }
+
+ /** Nothing to do */
+ @Override
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
+
+ private static int getSourceNonTerminalPosition(final int[] words, int nonTerminalIndex) {
+ int nonTerminalsSeen = 0;
+ for (int i = 0; i < words.length; i++) {
+ if (nt(words[i])) {
+ nonTerminalsSeen++;
+ if (nonTerminalsSeen == nonTerminalIndex) {
+ return i;
+ }
+ }
+ }
+ throw new RuntimeException(
+ String.format(
+ "Can not find %s-th non terminal in source ids: %s. This should not happen!",
+ nonTerminalIndex,
+ arrayToString(words)));
+ }
+
+ /**
+ * Returns the position of the nonTerminalIndex-th nonTerminal words.
+ * Non-terminals on target sides of rules are indexed by
+ * their order on the source side, e.g. '-1', '-2',
+ * Thus, if index==0 we return the index of '-1'.
+ * For index==1, we return index of '-2'
+ */
+ private static int getTargetNonTerminalPosition(int[] words, int nonTerminalIndex) {
+ for (int pos = 0; pos < words.length; pos++) {
+ if (nt(words[pos]) && -(words[pos] + 1) == nonTerminalIndex) {
+ return pos;
+ }
+ }
+ throw new RuntimeException(
+ String.format(
+ "Can not find %s-th non terminal in target ids: %s. This should not happen!",
+ nonTerminalIndex,
+ arrayToString(words)));
+ }
+
+ private static String arrayToString(int[] ids) {
+ StringBuilder sb = new StringBuilder();
+ for (int i : ids) {
+ sb.append(i + " ");
+ }
+ return sb.toString().trim();
+ }
+
+ private void substituteNonTerminal(
+ final OutputString parentState,
+ final OutputString childState) {
+ int mergePosition;
+ if (extractSource) {
+ /* correct nonTerminal is given by the tailNodePosition of the childState (zero-index, thus +1) and
+ * current parentState's arity. If the parentState has already filled one of two available slots,
+ * we need to use the remaining one, even if childState refers to the second slot.
+ */
+ mergePosition = getSourceNonTerminalPosition(
+ parentState.words, min(childState.tailNodePosition + 1, parentState.arity));
+ } else {
+ mergePosition = getTargetNonTerminalPosition(
+ parentState.words, childState.tailNodePosition);
+ }
+ parentState.substituteNonTerminalAtPosition(childState.words, mergePosition);
+ }
+
+ private void merge(final OutputString state) {
+ if (!outputStringStack.isEmpty()
+ && state.arity == 0) {
+ if (outputStringStack.peek().arity == 0) {
+ throw new IllegalStateException("Parent OutputString has arity of 0. Cannot merge.");
+ }
+ final OutputString parent = outputStringStack.pop();
+ substituteNonTerminal(parent, state);
+ merge(parent);
+ } else {
+ outputStringStack.add(state);
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (outputStringStack.isEmpty()) {
+ return "";
+ }
+
+ if (outputStringStack.size() != 1) {
+ throw new IllegalStateException(
+ String.format(
+ "Stack should contain only a single (last) element, but was size %d", outputStringStack.size()));
+ }
+ return getWords(outputStringStack.pop().words);
+ }
+
+ /** Stores necessary information to obtain an output string on source or target side */
+ private class OutputString {
+
+ private int[] words;
+ private int arity;
+ private final int tailNodePosition;
+
+ private OutputString(int[] words, int arity, int tailNodePosition) {
+ this.words = words;
+ this.arity = arity;
+ this.tailNodePosition = tailNodePosition;
+ }
+
+ /**
+ * Merges child words into this at the correct
+ * non terminal position of this.
+ * The correct position is determined by the tailNodePosition
+ * of child and the arity of this.
+ */
+ private void substituteNonTerminalAtPosition(final int[] words, final int position) {
+ assert(nt(this.words[position]));
+ final int[] result = new int[words.length + this.words.length - 1];
+ int resultIndex = 0;
+ for (int i = 0; i < position; i++) {
+ result[resultIndex++] = this.words[i];
+ }
+ for (int i = 0; i < words.length; i++) {
+ result[resultIndex++] = words[i];
+ }
+ for (int i = position + 1; i < this.words.length; i++) {
+ result[resultIndex++] = this.words[i];
+ }
+ // update words and reduce arity of this OutputString
+ this.words = result;
+ arity--;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/ViterbiExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/ViterbiExtractor.java b/src/joshua/decoder/hypergraph/ViterbiExtractor.java
index 1137792..31c8dc0 100644
--- a/src/joshua/decoder/hypergraph/ViterbiExtractor.java
+++ b/src/joshua/decoder/hypergraph/ViterbiExtractor.java
@@ -18,11 +18,14 @@
*/
package joshua.decoder.hypergraph;
+import static java.util.Collections.emptyList;
+
import java.util.ArrayList;
import java.util.List;
-import joshua.corpus.Vocabulary;
-import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.ff.FeatureFunction;
+import joshua.decoder.ff.FeatureVector;
+import joshua.decoder.segment_file.Sentence;
/**
* @author Zhifei Li, <zh...@gmail.com>
@@ -31,64 +34,91 @@ import joshua.decoder.ff.tm.Rule;
public class ViterbiExtractor {
- // get one-best string under item
- public static String extractViterbiString(HGNode node) {
- StringBuffer res = new StringBuffer();
-
- HyperEdge edge = node.bestHyperedge;
- Rule rl = edge.getRule();
-
- if (null == rl) { // deductions under "goal item" does not have rule
- if (edge.getTailNodes().size() != 1) {
- throw new RuntimeException("deduction under goal item have not equal one item");
- }
- return extractViterbiString(edge.getTailNodes().get(0));
- }
- int[] english = rl.getEnglish();
- for (int c = 0; c < english.length; c++) {
- if (Vocabulary.nt(english[c])) {
- int id = -(english[c] + 1);
- HGNode child = (HGNode) edge.getTailNodes().get(id);
- res.append(extractViterbiString(child));
- } else {
- res.append(Vocabulary.word(english[c]));
+ /**
+ * This function recursively visits the nodes of the Viterbi derivation in a depth-first
+ * traversal, applying the walker to each of the nodes. It provides a more general framework for
+ * implementing operations on a tree.
+ *
+ * @param node the node to start viterbi traversal from
+ * @param walker an implementation of the WalkerFunction interface, to be applied to each node in
+ * the tree
+ * @param nodeIndex the tail node index of the given node. This allows implementations of the
+ * WalkerFunction to associate nonTerminals with the index of node in the outgoing edges
+ * list of tail nodes.
+ */
+ public static void viterbiWalk(
+ final HGNode node,
+ final WalkerFunction walker,
+ final int nodeIndex) {
+ // apply the walking function to the node
+ walker.apply(node, nodeIndex);
+ // recurse on the anterior nodes of the best hyperedge in source order
+ final HyperEdge bestEdge = node.bestHyperedge;
+ final List<HGNode> tailNodes = bestEdge.getTailNodes();
+ if (tailNodes != null) {
+ for (int tailNodeIndex = 0; tailNodeIndex < tailNodes.size(); tailNodeIndex++) {
+ viterbiWalk(tailNodes.get(tailNodeIndex), walker, tailNodeIndex);
}
- if (c < english.length - 1) res.append(' ');
}
- return res.toString();
}
- public static String extractViterbiAlignment(HGNode node) {
- WordAlignmentState viterbiAlignment = buildViterbiAlignment(node);
- return viterbiAlignment.toFinalString();
+ public static void viterbiWalk(final HGNode node, final WalkerFunction walker) {
+ viterbiWalk(node, walker, 0);
}
- // get one-best alignment for Viterbi string
- public static WordAlignmentState buildViterbiAlignment(HGNode node) {
- HyperEdge edge = node.bestHyperedge;
- Rule rl = edge.getRule();
- if (rl == null) { // deductions under "goal item" does not have rule
- if (edge.getTailNodes().size() != 1)
- throw new RuntimeException("deduction under goal item have not equal one item");
- return buildViterbiAlignment(edge.getTailNodes().get(0));
- }
- WordAlignmentState waState = new WordAlignmentState(rl, node.i);
- if (edge.getTailNodes() != null) {
- int[] english = rl.getEnglish();
- for (int c = 0; c < english.length; c++) {
- if (Vocabulary.nt(english[c])) {
- // determines the index in the tail node array by
- // the index of the nonterminal in the source [english[c] gives a negative
- // int]
- int index = -(english[c] + 1);
- waState.substituteIn(buildViterbiAlignment(edge.getTailNodes().get(index)));
- }
- }
- }
- return waState;
+ /**
+ * Returns the Viterbi translation of the Hypergraph (includes sentence markers)
+ */
+ public static String getViterbiString(final HyperGraph hg) {
+ if (hg == null)
+ return "";
+
+ final WalkerFunction viterbiOutputStringWalker = new OutputStringExtractor(false);
+ viterbiWalk(hg.goalNode, viterbiOutputStringWalker);
+ return viterbiOutputStringWalker.toString();
}
-
- // ######## find 1best hypergraph#############
+
+ /**
+ * Returns the Viterbi feature vector
+ */
+ public static FeatureVector getViterbiFeatures(
+ final HyperGraph hg,
+ final List<FeatureFunction> featureFunctions,
+ final Sentence sentence) {
+ if (hg == null)
+ return new FeatureVector();
+
+ final FeatureVectorExtractor extractor = new FeatureVectorExtractor(
+ featureFunctions, sentence);
+ viterbiWalk(hg.goalNode, extractor);
+ return extractor.getFeatures();
+ }
+
+ /**
+ * Returns the Viterbi Word Alignments as String.
+ */
+ public static String getViterbiWordAlignments(final HyperGraph hg) {
+ if (hg == null)
+ return "";
+
+ final WordAlignmentExtractor wordAlignmentWalker = new WordAlignmentExtractor();
+ viterbiWalk(hg.goalNode, wordAlignmentWalker);
+ return wordAlignmentWalker.toString();
+ }
+
+ /**
+ * Returns the Viterbi Word Alignments as list of lists (target-side).
+ */
+ public static List<List<Integer>> getViterbiWordAlignmentList(final HyperGraph hg) {
+ if (hg == null)
+ return emptyList();
+
+ final WordAlignmentExtractor wordAlignmentWalker = new WordAlignmentExtractor();
+ viterbiWalk(hg.goalNode, wordAlignmentWalker);
+ return wordAlignmentWalker.getFinalWordAlignments();
+ }
+
+ /** find 1best hypergraph */
public static HyperGraph getViterbiTreeHG(HyperGraph hg_in) {
HyperGraph res =
new HyperGraph(cloneNodeWithBestHyperedge(hg_in.goalNode), -1, -1, null);
@@ -97,28 +127,6 @@ public class ViterbiExtractor {
return res;
}
- /**
- * This function recursively visits the nodes of the Viterbi derivation in a depth-first
- * traversal, applying the walker to each of the nodes. It provides a more general framework for
- * implementing operations on a tree.
- *
- * @param node the node to start traversal from
- * @param walker an implementation of the ViterbieWalker interface, to be applied to each node in
- * the tree
- */
- public static void walk(HGNode node, WalkerFunction walker) {
- // apply the walking function to the node
- walker.apply(node);
-
- // recurse on the anterior nodes of the best hyperedge
- HyperEdge bestEdge = node.bestHyperedge;
- if (null != bestEdge.getTailNodes()) {
- for (HGNode antNode : bestEdge.getTailNodes()) {
- walk(antNode, walker);
- }
- }
- }
-
private static void get1bestTreeNode(HGNode it) {
HyperEdge dt = it.bestHyperedge;
if (null != dt.getTailNodes()) {
@@ -151,5 +159,4 @@ public class ViterbiExtractor {
antNodes, inEdge.getSourcePath());
return res;
}
- // ###end
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/ViterbiFeatureVectorWalkerFunction.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/ViterbiFeatureVectorWalkerFunction.java b/src/joshua/decoder/hypergraph/ViterbiFeatureVectorWalkerFunction.java
deleted file mode 100644
index 5af6c4d..0000000
--- a/src/joshua/decoder/hypergraph/ViterbiFeatureVectorWalkerFunction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package joshua.decoder.hypergraph;
-
-import static joshua.decoder.chart_parser.ComputeNodeResult.computeTransitionFeatures;
-
-import java.util.List;
-import java.util.Map;
-
-import joshua.decoder.ff.FeatureFunction;
-import joshua.decoder.ff.FeatureVector;
-import joshua.decoder.segment_file.Sentence;
-
-public class ViterbiFeatureVectorWalkerFunction implements WalkerFunction {
-
- private final FeatureVector features;
- private final List<FeatureFunction> featureFunctions;
- private final Sentence sourceSentence;
-
- public ViterbiFeatureVectorWalkerFunction(
- final List<FeatureFunction> featureFunctions,
- final Sentence sourceSentence) {
- this.features = new FeatureVector();
- this.featureFunctions = featureFunctions;
- this.sourceSentence = sourceSentence;
- }
-
- /**
- * Recompute feature values for each Viterbi edge and add to features.
- */
- @Override
- public void apply(HGNode node) {
- final FeatureVector edgeFeatures = computeTransitionFeatures(
- featureFunctions, node.bestHyperedge, node.i, node.j, sourceSentence);
- features.add(edgeFeatures);
- }
-
- public FeatureVector getFeatures() {
- return features;
- }
-
- public Map<String,Float> getFeaturesMap() {
- return features.getMap();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/ViterbiOutputStringWalkerFunction.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/ViterbiOutputStringWalkerFunction.java b/src/joshua/decoder/hypergraph/ViterbiOutputStringWalkerFunction.java
deleted file mode 100644
index 0c84375..0000000
--- a/src/joshua/decoder/hypergraph/ViterbiOutputStringWalkerFunction.java
+++ /dev/null
@@ -1,96 +0,0 @@
-package joshua.decoder.hypergraph;
-
-import static java.lang.Integer.MAX_VALUE;
-import static joshua.corpus.Vocabulary.getWords;
-import static joshua.corpus.Vocabulary.nt;
-
-import java.util.Stack;
-
-import joshua.decoder.ff.tm.Rule;
-
-public class ViterbiOutputStringWalkerFunction implements WalkerFunction {
-
- private Stack<int[]> viterbiWords = new Stack<int[]>();
-
- @Override
- public void apply(HGNode node) {
- final Rule rule = node.bestHyperedge.getRule();
- if (rule != null) {
- merge(rule.getEnglish());
- }
- }
-
- private boolean containsNonTerminals(final int[] ids) {
- boolean hasNonTerminals = false;
- for (int i = 0; i < ids.length; i++) {
- if (nt(ids[i])) {
- hasNonTerminals = true;
- break;
- }
- }
- return hasNonTerminals;
- }
-
- /**
- * Returns the index of the next non-terminal slot to fill.
- * Since non-terminals in right hand sides of rules are indexed by
- * their order on the source side, this function looks for the largest
- * negative id in ids and returns its index.
- */
- private int getNextNonTerminalIndexToFill(final int[] ids) {
- int nextIndex = 0;
- int nextNonTerminal = -MAX_VALUE;
- for (int i = 0; i < ids.length; i++) {
- if (nt(ids[i]) && ids[i] > nextNonTerminal) {
- nextIndex = i;
- nextNonTerminal = ids[i];
- }
- }
- return nextIndex;
- }
-
- private int[] substituteNonTerminal(final int[] parentWords, final int[] childWords) {
- final int ntIndex = getNextNonTerminalIndexToFill(parentWords);
- final int[] result = new int[parentWords.length + childWords.length - 1];
- int resultIndex = 0;
- for (int i = 0; i < ntIndex; i++) {
- result[resultIndex++] = parentWords[i];
- }
- for (int i = 0; i < childWords.length; i++) {
- result[resultIndex++] = childWords[i];
- }
- for (int i = ntIndex + 1; i < parentWords.length; i++) {
- result[resultIndex++] = parentWords[i];
- }
- return result;
- }
-
- private void merge(final int[] words) {
- if (!containsNonTerminals(words)
- && !viterbiWords.isEmpty()
- && containsNonTerminals(viterbiWords.peek())) {
- merge(substituteNonTerminal(viterbiWords.pop(), words));
- } else {
- viterbiWords.add(words);
- }
- }
-
- @Override
- public String toString() {
- if (viterbiWords.isEmpty()) {
- return "";
- }
-
- if (viterbiWords.size() != 1) {
- throw new RuntimeException(
- String.format(
- "Stack of ViterbiOutputStringWalker should contain only a single (last) element, but was size %d", viterbiWords.size()));
- }
-
- String result = getWords(viterbiWords.peek());
- // strip of sentence markers (<s>,</s>)
- result = result.substring(result.indexOf(' ') + 1, result.lastIndexOf(' '));
- return result.trim();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/WalkerFunction.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/WalkerFunction.java b/src/joshua/decoder/hypergraph/WalkerFunction.java
index d46ef50..65bffbf 100644
--- a/src/joshua/decoder/hypergraph/WalkerFunction.java
+++ b/src/joshua/decoder/hypergraph/WalkerFunction.java
@@ -20,10 +20,15 @@ package joshua.decoder.hypergraph;
/**
* Classes implementing this interface define a single function that is applied to each node. This
- * interface is used for various walkers (ViterbiExtractor, ForestWalker).
+ * interface is used for various walkers (ViterbiExtractor).
*/
public interface WalkerFunction {
- void apply(HGNode node);
+ /**
+ * Function that is applied to node at tail node index nodeIndex.
+ * nodeIndex indicates the index of node in the list of tailnodes for the
+ * outgoing edge.
+ */
+ void apply(HGNode node, int nodeIndex);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/WordAlignmentExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/WordAlignmentExtractor.java b/src/joshua/decoder/hypergraph/WordAlignmentExtractor.java
index 2dfbcb2..837c69f 100644
--- a/src/joshua/decoder/hypergraph/WordAlignmentExtractor.java
+++ b/src/joshua/decoder/hypergraph/WordAlignmentExtractor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package joshua.decoder.hypergraph;
import static java.util.Collections.emptyList;
@@ -29,9 +47,11 @@ public class WordAlignmentExtractor implements WalkerFunction, DerivationVisitor
// if alignment state has no NTs left AND stack is not empty
// and parent state on stack still needs something to substitute
if (!stack.isEmpty()
- && state.isComplete()
- && !stack.peek().isComplete()) {
+ && state.isComplete()) {
final WordAlignmentState parentState = stack.pop();
+ if (parentState.isComplete()) {
+ throw new IllegalStateException("Parent state already complete");
+ }
parentState.substituteIn(state);
merge(parentState);
} else {
@@ -44,7 +64,7 @@ public class WordAlignmentExtractor implements WalkerFunction, DerivationVisitor
*/
private void extract(final Rule rule, final int spanStart) {
if (rule != null) {
- merge (new WordAlignmentState(rule, spanStart));
+ merge(new WordAlignmentState(rule, spanStart));
}
}
@@ -53,7 +73,7 @@ public class WordAlignmentExtractor implements WalkerFunction, DerivationVisitor
* for best hyperedge from given node.
*/
@Override
- public void apply(HGNode node) {
+ public void apply(HGNode node, int nodeIndex) {
extract(node.bestHyperedge.getRule(), node.i);
}
@@ -63,7 +83,7 @@ public class WordAlignmentExtractor implements WalkerFunction, DerivationVisitor
* the Derivation state.
*/
@Override
- public void before(final DerivationState state, final int level) {
+ public void before(final DerivationState state, final int level, int tailNodeIndex) {
extract(state.edge.getRule(), state.parentNode.i);
}
@@ -71,7 +91,7 @@ public class WordAlignmentExtractor implements WalkerFunction, DerivationVisitor
* Nothing to do after visiting a node.
*/
@Override
- public void after(final DerivationState state, final int level) {}
+ public void after(final DerivationState state, final int level, int tailNodeIndex) {}
/**
* Final word alignment without sentence markers
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/hypergraph/WordAlignmentState.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/hypergraph/WordAlignmentState.java b/src/joshua/decoder/hypergraph/WordAlignmentState.java
index d47fa38..258e062 100644
--- a/src/joshua/decoder/hypergraph/WordAlignmentState.java
+++ b/src/joshua/decoder/hypergraph/WordAlignmentState.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package joshua.decoder.hypergraph;
import java.util.ArrayList;
@@ -125,7 +143,7 @@ public class WordAlignmentState {
* substitutes a child WorldAlignmentState into this instance at the first
* NT it finds. Also shifts the indeces in this instance by the span/width of the
* child that is to be substituted.
- * Substitution order is determined by the architecture of Joshua's hypergraph.
+ * Substitution order is determined by the source-first traversal through the hypergraph.
*/
void substituteIn(WordAlignmentState child) {
// update existing indexes by length of child (has no effect on NULL and
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/decoder/io/JSONMessage.java
----------------------------------------------------------------------
diff --git a/src/joshua/decoder/io/JSONMessage.java b/src/joshua/decoder/io/JSONMessage.java
index 6b6b77f..2733db4 100644
--- a/src/joshua/decoder/io/JSONMessage.java
+++ b/src/joshua/decoder/io/JSONMessage.java
@@ -90,7 +90,7 @@ public class JSONMessage {
JSONMessage message = new JSONMessage();
String[] results = translation.toString().split("\\n");
if (results.length > 0) {
- JSONMessage.TranslationItem item = message.addTranslation(translation.rawTranslation());
+ JSONMessage.TranslationItem item = message.addTranslation(translation.getStructuredTranslation().getTranslationString());
for (String result: results) {
String[] tokens = result.split(" \\|\\|\\| ");
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/src/joshua/oracle/OracleExtractionHG.java
----------------------------------------------------------------------
diff --git a/src/joshua/oracle/OracleExtractionHG.java b/src/joshua/oracle/OracleExtractionHG.java
index 9c32278..7e7fcb8 100644
--- a/src/joshua/oracle/OracleExtractionHG.java
+++ b/src/joshua/oracle/OracleExtractionHG.java
@@ -18,6 +18,9 @@
*/
package joshua.oracle;
+import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static joshua.util.FormatUtils.removeSentenceMarkers;
+
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.ArrayList;
@@ -31,7 +34,6 @@ import joshua.decoder.hypergraph.HGNode;
import joshua.decoder.hypergraph.HyperEdge;
import joshua.decoder.hypergraph.HyperGraph;
import joshua.decoder.hypergraph.KBestExtractor;
-import joshua.decoder.hypergraph.ViterbiExtractor;
import joshua.util.FileUtility;
import joshua.util.io.LineReader;
@@ -175,7 +177,7 @@ public class OracleExtractionHG extends SplitHg {
orc_bleu = (Double) res[1];
} else {
HyperGraph hg_oracle = orc_extractor.oracle_extract_hg(hg, hg.sentLen(), lm_order, ref_sent);
- orc_sent = ViterbiExtractor.extractViterbiString(hg_oracle.goalNode);
+ orc_sent = removeSentenceMarkers(getViterbiString(hg_oracle));
orc_bleu = orc_extractor.get_best_goal_cost(hg, orc_extractor.g_tbl_split_virtual_items);
time_on_orc_extract += System.currentTimeMillis() - start_time;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/test/bn-en/hiero/input.bn
----------------------------------------------------------------------
diff --git a/test/bn-en/hiero/input.bn b/test/bn-en/hiero/input.bn
index befea75..be6a92b 100644
--- a/test/bn-en/hiero/input.bn
+++ b/test/bn-en/hiero/input.bn
@@ -74,7 +74,7 @@
ভুট্টো এ্যাসেম্বলি বয়কট করার হুমকি দিয়ে ঘোষণা দেন যে ইয়াহিয়া খান মুজিবকে সরকার গঠনের জন্য আহ্বান জানালে তিনি সে সরকারকে মেনে নেবেন না ।
আর computer শব্দের অর্থ গণনাকারী যন্ত্র ।
১৭৭৬ সালের ৪ জুলাই এই উপনিবেশগুলি একটি স্বাধীনতার ঘোষণাপত্র জারি করে ।
-জার্মানি -lrb- জার্মান ভাষায় : deutschland ডয়চ্ লান্ট্ আ-ধ্ব-ব : -lsb- dɔʏtʃlant -rsb- -rrb- মধ্য ইউরোপের একটি রাষ্ট্র ।
+জার্মানি -lrb- জার্মান ভাষায় : deutschland ডয়চ্ লান্ট্ আ-ধ্ব-ব : [ dɔʏtʃlant ] -rrb- মধ্য ইউরোপের একটি রাষ্ট্র ।
খ্রিস্টধর্ম রাশিয়ার প্রধান ধর্ম ।
কিন্তু গলদের শিক্ষার রোমানীকরণের গতি ছিল ধীর ।
বিষয়শ্রেণী : গনু ফাউন্ডেশন
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5b791285/tst/joshua/system/StructuredTranslationTest.java
----------------------------------------------------------------------
diff --git a/tst/joshua/system/StructuredTranslationTest.java b/tst/joshua/system/StructuredTranslationTest.java
index 821ceea..e14fae2 100644
--- a/tst/joshua/system/StructuredTranslationTest.java
+++ b/tst/joshua/system/StructuredTranslationTest.java
@@ -1,6 +1,7 @@
package joshua.system;
import static java.util.Arrays.asList;
+import static joshua.decoder.ff.FeatureVector.DENSE_FEATURE_NAMES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -13,6 +14,7 @@ import joshua.decoder.Decoder;
import joshua.decoder.JoshuaConfiguration;
import joshua.decoder.StructuredTranslation;
import joshua.decoder.Translation;
+import joshua.decoder.ff.FeatureVector;
import joshua.decoder.segment_file.Sentence;
import org.junit.After;
@@ -56,7 +58,6 @@ public class StructuredTranslationTest {
@Before
public void setUp() throws Exception {
- Vocabulary.clear();
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.search_algorithm = "cky";
joshuaConfig.mark_oovs = false;
@@ -76,7 +77,7 @@ public class StructuredTranslationTest {
joshuaConfig.weights.add("tm_pt_4 1");
joshuaConfig.weights.add("tm_pt_5 1");
joshuaConfig.weights.add("tm_glue_0 1");
- joshuaConfig.weights.add("OOVPenalty 2");
+ joshuaConfig.weights.add("OOVPenalty 1");
decoder = new Decoder(joshuaConfig, ""); // second argument (configFile
// is not even used by the
// constructor/initialize)
@@ -84,7 +85,6 @@ public class StructuredTranslationTest {
@After
public void tearDown() throws Exception {
- Vocabulary.clear();
decoder.cleanUp();
decoder = null;
}
@@ -106,9 +106,24 @@ public class StructuredTranslationTest {
// THEN
assertEquals(EXPECTED_TRANSLATION + " | " + EXPECTED_WORD_ALIGNMENT_STRING, translation);
}
+
+ @Test
+ public void givenInput_whenRegularOutputFormatWithTopN1_thenExpectedOutput() {
+ // GIVEN
+ joshuaConfig.construct_structured_output = false;
+ joshuaConfig.outputFormat = "%s | %e | %a | %c";
+ joshuaConfig.topN = 1;
+
+ // WHEN
+ final String translation = decode(INPUT).toString().trim();
+
+ // THEN
+ assertEquals(EXPECTED_TRANSLATION + " | " + INPUT + " | " + EXPECTED_WORD_ALIGNMENT_STRING + String.format(" | %.3f", EXPECTED_SCORE),
+ translation);
+ }
@Test
- public void givenInput_whenSaarStructuredOutputFormat_thenExpectedOutput() {
+ public void givenInput_whenStructuredOutputFormat_thenExpectedOutput() {
// GIVEN
joshuaConfig.construct_structured_output = true;
@@ -130,7 +145,7 @@ public class StructuredTranslationTest {
}
@Test
- public void givenEmptyInput_whenSaarStructuredOutputFormat_thenEmptyOutput() {
+ public void givenEmptyInput_whenStructuredOutputFormat_thenEmptyOutput() {
// GIVEN
joshuaConfig.construct_structured_output = true;
@@ -149,7 +164,7 @@ public class StructuredTranslationTest {
}
@Test
- public void givenOOVInput_whenSaarStructuredOutputFormat_thenOOVOutput() {
+ public void givenOOVInput_whenStructuredOutputFormat_thenOOVOutput() {
// GIVEN
joshuaConfig.construct_structured_output = true;
final String input = "gabarbl";
@@ -164,7 +179,7 @@ public class StructuredTranslationTest {
// THEN
assertEquals(input, translationString);
assertTrue(translatedTokens.contains(input));
- assertEquals(-199.0, translationScore, 0.00001);
+ assertEquals(-99.0, translationScore, 0.00001);
assertTrue(wordAlignment.contains(asList(0)));
}