You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/01 02:51:54 UTC
[59/94] [abbrv] incubator-joshua git commit: Merge branch 'master'
into JOSHUA-252 (compiling, but not tested)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
index 9e1b06c,0000000..a8cd161
mode 100644,000000..100644
--- a/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
@@@ -1,227 -1,0 +1,228 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
++import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.io.LineReader;
+
+/***
+ * The RuleBigram feature is an indicator feature that counts target word bigrams that are created when
+ * a rule is applied. It accepts three parameters:
+ *
+ * -vocab /path/to/vocab
+ *
+ * The path to a vocabulary, where each line is of the format ID WORD COUNT.
+ *
+ * -threshold N
+ *
+ * Mask to UNK all words whose COUNT is less than N.
+ *
+ * -top-n N
+ *
+ * Only use the top N words.
+ */
+
+public class TargetBigram extends StatefulFF {
+
+ private HashSet<String> vocab = null;
+ private int maxTerms = 1000000;
+ private int threshold = 0;
+
+ public TargetBigram(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "TargetBigram", args, config);
+
+ if (parsedArgs.containsKey("threshold"))
+ threshold = Integer.parseInt(parsedArgs.get("threshold"));
+
+ if (parsedArgs.containsKey("top-n"))
+ maxTerms = Integer.parseInt(parsedArgs.get("top-n"));
+
+ if (parsedArgs.containsKey("vocab")) {
+ loadVocab(parsedArgs.get("vocab"));
+ }
+ }
+
+ /**
+ * Load vocabulary items passing the 'threshold' and 'top-n' filters.
+ *
+ * @param filename
+ */
+ private void loadVocab(String filename) {
+ this.vocab = new HashSet<String>();
+ this.vocab.add("<s>");
+ this.vocab.add("</s>");
+ try {
+ LineReader lineReader = new LineReader(filename);
+ for (String line: lineReader) {
+ if (lineReader.lineno() > maxTerms)
+ break;
+
+ String[] tokens = line.split("\\s+");
+ String word = tokens[1];
+ int count = Integer.parseInt(tokens[2]);
+
+ if (count >= threshold)
+ vocab.add(word);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(String.format(
+ "* FATAL: couldn't load TargetBigram vocabulary '%s'", filename), e);
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int spanStart, int spanEnd,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc) {
+
+ int[] enWords = rule.getEnglish();
+
+ int left = -1;
+ int right = -1;
+
+ List<String> currentNgram = new LinkedList<String>();
+ for (int c = 0; c < enWords.length; c++) {
+ int curID = enWords[c];
+
- if (Vocabulary.nt(curID)) {
++ if (FormatUtils.isNonterminal(curID)) {
+ int index = -(curID + 1);
+ NgramDPState state = (NgramDPState) tailNodes.get(index).getDPState(stateIndex);
+ int[] leftContext = state.getLeftLMStateWords();
+ int[] rightContext = state.getRightLMStateWords();
+
+ // Left context.
+ for (int token : leftContext) {
+ currentNgram.add(getWord(token));
+ if (left == -1)
+ left = token;
+ right = token;
+ if (currentNgram.size() == 2) {
+ String ngram = join(currentNgram);
+ acc.add(String.format("%s_%s", name, ngram), 1);
+ // System.err.println(String.format("ADDING %s_%s", name, ngram));
+ currentNgram.remove(0);
+ }
+ }
+ // Replace right context.
+ int tSize = currentNgram.size();
+ for (int i = 0; i < rightContext.length; i++)
+ currentNgram.set(tSize - rightContext.length + i, getWord(rightContext[i]));
+
+ } else { // terminal words
+ currentNgram.add(getWord(curID));
+ if (left == -1)
+ left = curID;
+ right = curID;
+ if (currentNgram.size() == 2) {
+ String ngram = join(currentNgram);
+ acc.add(String.format("%s_%s", name, ngram), 1);
+ // System.err.println(String.format("ADDING %s_%s", name, ngram));
+ currentNgram.remove(0);
+ }
+ }
+ }
+
+ NgramDPState state = new NgramDPState(new int[] { left }, new int[] { right });
+ // System.err.println(String.format("RULE %s -> state %s", rule.getRuleString(), state));
+ return state;
+ }
+
+ /**
+ * Returns the word after comparing against the private vocabulary (if set).
+ *
+ * @param curID
+ * @return the word
+ */
+ private String getWord(int curID) {
+ String word = Vocabulary.word(curID);
+
+ if (vocab != null && ! vocab.contains(word)) {
+ return "UNK";
+ }
+
+ return word;
+ }
+
+ /**
+ * We don't compute a future cost.
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * There is nothing to be done here, since <s> and </s> are included in rules that are part
+ * of the grammar. We simply return the DP state of the tail node.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ return tailNode.getDPState(stateIndex);
+ }
+
+ /**
+ * TargetBigram features are only computed across hyperedges, so there is nothing to be done here.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * Join a list with the _ character. I am sure this is in a library somewhere.
+ *
+ * @param list a list of strings
+ * @return the joined String
+ */
+ private String join(List<String> list) {
+ StringBuilder sb = new StringBuilder();
+ for (String item : list) {
+ sb.append(item.toString() + "_");
+ }
+
+ return sb.substring(0, sb.length() - 1);
+ }
+
+ @Override
+ public double estimateLogP(Rule rule, int sentID) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public double getWeight() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
index 47410f4,0000000..3d636d9
mode 100644,000000..100644
--- a/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
@@@ -1,538 -1,0 +1,539 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.primitives.Ints;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Support;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.lm.berkeley_lm.LMGrammarBerkeley;
+import org.apache.joshua.decoder.ff.lm.KenLM;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
++import org.apache.joshua.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class performs the following:
+ * <ol>
+ * <li>Gets the additional LM score due to combinations of small items into larger ones by using
+ * rules</li>
+ * <li>Gets the LM state</li>
+ * <li>Gets the left-side LM state estimation score</li>
+ * </ol>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class LanguageModelFF extends StatefulFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LanguageModelFF.class);
+
+ public static int LM_INDEX = 0;
+ private int startSymbolId;
+
+ /**
+ * N-gram language model. We assume the language model is in ARPA format for equivalent state:
+ *
+ * <ol>
+ * <li>We assume it is a backoff lm, and high-order ngram implies low-order ngram; absense of
+ * low-order ngram implies high-order ngram</li>
+ * <li>For a ngram, existence of backoffweight => existence a probability Two ways of dealing with
+ * low counts:
+ * <ul>
+ * <li>SRILM: don't multiply zeros in for unknown words</li>
+ * <li>Pharaoh: cap at a minimum score exp(-10), including unknown words</li>
+ * </ul>
+ * </li>
+ * </ol>
+ */
+ protected NGramLanguageModel languageModel;
+
+ /**
+ * We always use this order of ngram, though the LMGrammar may provide higher order probability.
+ */
+ protected final int ngramOrder;
+
+ /*
+ * We cache the weight of the feature since there is only one.
+ */
+ protected float weight;
+ protected String type;
+ protected String path;
+
+ /* Whether this is a class-based LM */
+ private boolean isClassLM;
+ private ClassMap classMap;
+
+ protected class ClassMap {
+
+ private final int OOV_id = Vocabulary.getUnknownId();
+ private HashMap<Integer, Integer> classMap;
+
+ public ClassMap(String file_name) throws IOException {
+ this.classMap = new HashMap<Integer, Integer>();
+ read(file_name);
+ }
+
+ public int getClassID(int wordID) {
+ return this.classMap.getOrDefault(wordID, OOV_id);
+ }
+
+ /**
+ * Reads a class map from file.
+ *
+ * @param file_name
+ * @throws IOException
+ */
+ private void read(String file_name) throws IOException {
+
+ int lineno = 0;
+ for (String line: new org.apache.joshua.util.io.LineReader(file_name, false)) {
+ lineno++;
+ String[] lineComp = line.trim().split("\\s+");
+ try {
+ this.classMap.put(Vocabulary.id(lineComp[0]), Vocabulary.id(lineComp[1]));
+ } catch (java.lang.ArrayIndexOutOfBoundsException e) {
+ LOG.warn("bad vocab line #{} '{}'", lineno, line);
+ LOG.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ }
+
+ public LanguageModelFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, String.format("lm_%d", LanguageModelFF.LM_INDEX++), args, config);
+
+ this.type = parsedArgs.get("lm_type");
+ this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
+ this.path = parsedArgs.get("lm_file");
+
+ if (parsedArgs.containsKey("class_map"))
+ try {
+ this.isClassLM = true;
+ this.classMap = new ClassMap(parsedArgs.get("class_map"));
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ // The dense feature initialization hasn't happened yet, so we have to retrieve this as sparse
+ this.weight = weights.getSparse(name);
+
+ initializeLM();
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Initializes the underlying language model.
+ */
+ protected void initializeLM() {
+ if (type.equals("kenlm")) {
+ this.languageModel = new KenLM(ngramOrder, path);
+
+ } else if (type.equals("berkeleylm")) {
+ this.languageModel = new LMGrammarBerkeley(ngramOrder, path);
+
+ } else {
+ String msg = String.format("* FATAL: Invalid backend lm_type '%s' for LanguageModel", type)
+ + "* Permissible values for 'lm_type' are 'kenlm' and 'berkeleylm'";
+ throw new RuntimeException(msg);
+ }
+
+ Vocabulary.registerLanguageModel(this.languageModel);
+ Vocabulary.id(config.default_non_terminal);
+
+ startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ }
+
+ public NGramLanguageModel getLM() {
+ return this.languageModel;
+ }
+
+ public String logString() {
+ if (languageModel != null)
+ return String.format("%s, order %d (weight %.3f)", name, languageModel.getOrder(), weight);
+ else
+ return "WHOA";
+ }
+
+ /**
+ * Computes the features incurred along this edge. Note that these features are unweighted costs
+ * of the feature; they are the feature cost, not the model cost, or the inner product of them.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ NgramDPState newState = null;
+ if (rule != null) {
+ if (config.source_annotations) {
+ // Get source side annotations and project them to the target side
+ newState = computeTransition(getTags(rule, i, j, sentence), tailNodes, acc);
+ }
+ else {
+ if (this.isClassLM) {
+ // Use a class language model
+ // Return target side classes
+ newState = computeTransition(getClasses(rule), tailNodes, acc);
+ }
+ else {
+ // Default LM
+ newState = computeTransition(rule.getEnglish(), tailNodes, acc);
+ }
+ }
+
+ }
+
+ return newState;
+ }
+
+ /**
+ * Input sentences can be tagged with information specific to the language model. This looks for
+ * such annotations by following a word's alignments back to the source words, checking for
+ * annotations, and replacing the surface word if such annotations are found.
+ * @param rule the {@link org.apache.joshua.decoder.ff.tm.Rule} to use
+ * @param begin todo
+ * @param end todo
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return todo
+ */
+ protected int[] getTags(Rule rule, int begin, int end, Sentence sentence) {
+ /* Very important to make a copy here, so the original rule is not modified */
+ int[] tokens = Arrays.copyOf(rule.getEnglish(), rule.getEnglish().length);
+ byte[] alignments = rule.getAlignment();
+
+ // System.err.println(String.format("getTags() %s", rule.getRuleString()));
+
+ /* For each target-side token, project it to each of its source-language alignments. If any of those
+ * are annotated, take the first annotation and quit.
+ */
+ if (alignments != null) {
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] > 0) { // skip nonterminals
+ for (int j = 0; j < alignments.length; j += 2) {
+ if (alignments[j] == i) {
+ String annotation = sentence.getAnnotation((int)alignments[i] + begin, "class");
+ if (annotation != null) {
+ // System.err.println(String.format(" word %d source %d abs %d annotation %d/%s",
+ // i, alignments[i], alignments[i] + begin, annotation, Vocabulary.word(annotation)));
+ tokens[i] = Vocabulary.id(annotation);
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return tokens;
+ }
+
+ /**
+ * Sets the class map if this is a class LM
+ * @param fileName a string path to a file
+ * @throws IOException if there is an error reading the input file
+ */
+ public void setClassMap(String fileName) throws IOException {
+ this.classMap = new ClassMap(fileName);
+ }
+
+ /**
+ * Replace each word in a rule with the target side classes.
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to use when obtaining tokens
+ * @return int[] of tokens
+ */
+ protected int[] getClasses(Rule rule) {
+ if (this.classMap == null) {
+ throw new RuntimeException("The class map is not set. Cannot use the class LM ");
+ }
+ /* Very important to make a copy here, so the original rule is not modified */
+ int[] tokens = Arrays.copyOf(rule.getEnglish(), rule.getEnglish().length);
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] > 0 ) {
+ tokens[i] = this.classMap.getClassID(tokens[i]);
+ }
+ }
+ return tokens;
+ }
+
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ return computeFinalTransition((NgramDPState) tailNode.getDPState(stateIndex), acc);
+ }
+
+ /**
+ * This function computes all the complete n-grams found in the rule, as well as the incomplete
+ * n-grams on the left-hand side.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+
+ float estimate = 0.0f;
+ boolean considerIncompleteNgrams = true;
+
+ int[] enWords = rule.getEnglish();
+
+ List<Integer> words = new ArrayList<Integer>();
+ boolean skipStart = (enWords[0] == startSymbolId);
+
+ /*
+ * Move through the words, accumulating language model costs each time we have an n-gram (n >=
+ * 2), and resetting the series of words when we hit a nonterminal.
+ */
+ for (int c = 0; c < enWords.length; c++) {
+ int currentWord = enWords[c];
- if (Vocabulary.nt(currentWord)) {
++ if (FormatUtils.isNonterminal(currentWord)) {
+ estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+ words.clear();
+ skipStart = false;
+ } else {
+ words.add(currentWord);
+ }
+ }
+ estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+
+ return weight * estimate;
+ }
+
+ /**
+ * Estimates the future cost of a rule. For the language model feature, this is the sum of the
+ * costs of the leftmost k-grams, k = [1..n-1].
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ NgramDPState state = (NgramDPState) currentState;
+
+ float estimate = 0.0f;
+ int[] leftContext = state.getLeftLMStateWords();
+
+ if (null != leftContext) {
+ boolean skipStart = true;
+ if (leftContext[0] != startSymbolId) {
+ skipStart = false;
+ }
+ estimate += scoreChunkLogP(leftContext, true, skipStart);
+ }
+ return weight * estimate;
+ }
+
+ /**
+ * Compute the cost of a rule application. The cost of applying a rule is computed by determining
+ * the n-gram costs for all n-grams created by this rule application, and summing them. N-grams
+ * are created when (a) terminal words in the rule string are followed by a nonterminal (b)
+ * terminal words in the rule string are preceded by a nonterminal (c) we encounter adjacent
+ * nonterminals. In all of these situations, the corresponding boundary words of the node in the
+ * hypergraph represented by the nonterminal must be retrieved.
+ *
+ * IMPORTANT: only complete n-grams are scored. This means that hypotheses with fewer words
+ * than the complete n-gram state remain *unscored*. This fact adds a lot of complication to the
+ * code, including the use of the computeFinal* family of functions, which correct this fact for
+ * sentences that are too short on the final transition.
+ */
+ private NgramDPState computeTransition(int[] enWords, List<HGNode> tailNodes, Accumulator acc) {
+
+ int[] current = new int[this.ngramOrder];
+ int[] shadow = new int[this.ngramOrder];
+ int ccount = 0;
+ float transitionLogP = 0.0f;
+ int[] left_context = null;
+
+ for (int c = 0; c < enWords.length; c++) {
+ int curID = enWords[c];
+
- if (Vocabulary.nt(curID)) {
++ if (FormatUtils.isNonterminal(curID)) {
+ int index = -(curID + 1);
+
+ NgramDPState state = (NgramDPState) tailNodes.get(index).getDPState(stateIndex);
+ int[] left = state.getLeftLMStateWords();
+ int[] right = state.getRightLMStateWords();
+
+ // Left context.
+ for (int i = 0; i < left.length; i++) {
+ current[ccount++] = left[i];
+
+ if (left_context == null && ccount == this.ngramOrder - 1)
+ left_context = Arrays.copyOf(current, ccount);
+
+ if (ccount == this.ngramOrder) {
+ // Compute the current word probability, and remove it.
+ float prob = this.languageModel.ngramLogProbability(current, this.ngramOrder);
+ // System.err.println(String.format("-> prob(%s) = %f", Vocabulary.getWords(current), prob));
+ transitionLogP += prob;
+ System.arraycopy(current, 1, shadow, 0, this.ngramOrder - 1);
+ int[] tmp = current;
+ current = shadow;
+ shadow = tmp;
+ --ccount;
+ }
+ }
+ System.arraycopy(right, 0, current, ccount - right.length, right.length);
+ } else { // terminal words
+ current[ccount++] = curID;
+
+ if (left_context == null && ccount == this.ngramOrder - 1)
+ left_context = Arrays.copyOf(current, ccount);
+
+ if (ccount == this.ngramOrder) {
+ // Compute the current word probability, and remove it.s
+ float prob = this.languageModel.ngramLogProbability(current, this.ngramOrder);
+ // System.err.println(String.format("-> prob(%s) = %f", Vocabulary.getWords(current), prob));
+ transitionLogP += prob;
+ System.arraycopy(current, 1, shadow, 0, this.ngramOrder - 1);
+ int[] tmp = current;
+ current = shadow;
+ shadow = tmp;
+ --ccount;
+ }
+ }
+ }
+ // acc.add(name, transitionLogP);
+ acc.add(denseFeatureIndex, transitionLogP);
+
+ if (left_context != null) {
+ return new NgramDPState(left_context, Arrays.copyOfRange(current, ccount - this.ngramOrder
+ + 1, ccount));
+ } else {
+ int[] context = Arrays.copyOf(current, ccount);
+ return new NgramDPState(context, context);
+ }
+ }
+
+ /**
+ * This function differs from regular transitions because we incorporate the cost of incomplete
+ * left-hand ngrams, as well as including the start- and end-of-sentence markers (if they were
+ * requested when the object was created).
+ *
+ * @param state the dynamic programming state
+ * @return the final transition probability (including incomplete n-grams)
+ */
+ private NgramDPState computeFinalTransition(NgramDPState state, Accumulator acc) {
+
+ // System.err.println(String.format("LanguageModel::computeFinalTransition()"));
+
+ float res = 0.0f;
+ LinkedList<Integer> currentNgram = new LinkedList<Integer>();
+ int[] leftContext = state.getLeftLMStateWords();
+ int[] rightContext = state.getRightLMStateWords();
+
+ for (int i = 0; i < leftContext.length; i++) {
+ int t = leftContext[i];
+ currentNgram.add(t);
+
+ if (currentNgram.size() >= 2) { // start from bigram
+ float prob = this.languageModel.ngramLogProbability(Support.toArray(currentNgram),
+ currentNgram.size());
+ res += prob;
+ }
+ if (currentNgram.size() == this.ngramOrder)
+ currentNgram.removeFirst();
+ }
+
+ // Tell the accumulator
+ // acc.add(name, res);
+ acc.add(denseFeatureIndex, res);
+
+ // State is the same
+ return new NgramDPState(leftContext, rightContext);
+ }
+
+
+ /**
+ * Compatibility method for {@link #scoreChunkLogP(int[], boolean, boolean)}
+ */
+ private float scoreChunkLogP(List<Integer> words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+ return scoreChunkLogP(Ints.toArray(words), considerIncompleteNgrams, skipStart);
+ }
+
+ /**
+ * This function is basically a wrapper for NGramLanguageModel::sentenceLogProbability(). It
+ * computes the probability of a phrase ("chunk"), using lower-order n-grams for the first n-1
+ * words.
+ *
+ * @param words
+ * @param considerIncompleteNgrams
+ * @param skipStart
+ * @return the phrase log probability
+ */
+ private float scoreChunkLogP(int[] words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+
+ float score = 0.0f;
+ if (words.length > 0) {
+ int startIndex;
+ if (!considerIncompleteNgrams) {
+ startIndex = this.ngramOrder;
+ } else if (skipStart) {
+ startIndex = 2;
+ } else {
+ startIndex = 1;
+ }
+ score = this.languageModel.sentenceLogProbability(words, this.ngramOrder, startIndex);
+ }
+
+ return score;
+ }
+
+ /**
+ * Public method to set LM_INDEX back to 0.
+ * Required if multiple instances of the JoshuaDecoder live in the same JVM.
+ */
+ public static void resetLmIndex() {
+ LM_INDEX = 0;
+ }
+
+ @Override
+ public double estimateLogP(Rule rule, int sentID) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public double getWeight() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
index 6869def,0000000..533365c
mode 100644,000000..100644
--- a/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
@@@ -1,201 -1,0 +1,202 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.lm.KenLM;
+import org.apache.joshua.decoder.ff.lm.KenLM.StateProbPair;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.KenLMState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
++import org.apache.joshua.util.FormatUtils;
+
+/**
+ * Wrapper for KenLM LMs with left-state minimization. We inherit from the regular
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+public class StateMinimizingLanguageModel extends LanguageModelFF {
+
+ // maps from sentence numbers to KenLM-side pools used to allocate state
+ private static final ConcurrentHashMap<Integer, Long> poolMap = new ConcurrentHashMap<Integer, Long>();
+
+ public StateMinimizingLanguageModel(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, args, config);
+ this.type = "kenlm";
+ if (parsedArgs.containsKey("lm_type") && ! parsedArgs.get("lm_type").equals("kenlm")) {
+ String msg = "* FATAL: StateMinimizingLanguageModel only supports 'kenlm' lm_type backend"
+ + "* Remove lm_type from line or set to 'kenlm'";
+ throw new RuntimeException(msg);
+ }
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Initializes the underlying language model.
+ */
+ @Override
+ public void initializeLM() {
+
+ // Override type (only KenLM supports left-state minimization)
+ this.languageModel = new KenLM(ngramOrder, path);
+
+ Vocabulary.registerLanguageModel(this.languageModel);
+ Vocabulary.id(config.default_non_terminal);
+
+ }
+
+ /**
+ * Estimates the cost of a rule. We override here since KenLM can do it more efficiently
+ * than the default {@link LanguageModelFF} class.
+ *
+ * Most of this function implementation is redundant with compute().
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+
+ int[] ruleWords = rule.getEnglish();
+
+ // The IDs we'll pass to KenLM
+ long[] words = new long[ruleWords.length];
+
+ for (int x = 0; x < ruleWords.length; x++) {
+ int id = ruleWords[x];
+
- if (Vocabulary.nt(id)) {
++ if (FormatUtils.isNonterminal(id)) {
+ // For the estimate, we can just mark negative values
+ words[x] = -1;
+
+ } else {
+ // Terminal: just add it
+ words[x] = id;
+ }
+ }
+
+ // Get the probability of applying the rule and the new state
+ return weight * ((KenLM) languageModel).estimateRule(words);
+ }
+
+ /**
+ * Computes the features incurred along this edge. Note that these features are unweighted costs
+ * of the feature; they are the feature cost, not the model cost, or the inner product of them.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ int[] ruleWords = config.source_annotations
+ ? getTags(rule, i, j, sentence)
+ : rule.getEnglish();
+
+ // The IDs we'll pass to KenLM
+ long[] words = new long[ruleWords.length];
+
+ for (int x = 0; x < ruleWords.length; x++) {
+ int id = ruleWords[x];
+
- if (Vocabulary.nt(id)) {
++ if (FormatUtils.isNonterminal(id)) {
+ // Nonterminal: retrieve the KenLM long that records the state
+ int index = -(id + 1);
+ KenLMState state = (KenLMState) tailNodes.get(index).getDPState(stateIndex);
+ words[x] = -state.getState();
+
+ } else {
+ // Terminal: just add it
+ words[x] = id;
+ }
+ }
+
+ int sentID = sentence.id();
+ // Since sentId is unique across threads, next operations are safe, but not atomic!
+ if (!poolMap.containsKey(sentID)) {
+ poolMap.put(sentID, KenLM.createPool());
+ }
+
+ // Get the probability of applying the rule and the new state
+ StateProbPair pair = ((KenLM) languageModel).probRule(words, poolMap.get(sentID));
+
+ // Record the prob
+// acc.add(name, pair.prob);
+ acc.add(denseFeatureIndex, pair.prob);
+
+ // Return the state
+ return pair.state;
+ }
+
+ /**
+ * Destroys the pool created to allocate state for this sentence. Called from the
+ * {@link org.apache.joshua.decoder.Translation} class after outputting the sentence or k-best list. Hosting
+ * this map here in KenLMFF statically allows pools to be shared across KenLM instances.
+ *
+ * @param sentId a key in the poolmap table to destroy
+ */
+ public void destroyPool(int sentId) {
+ if (poolMap.containsKey(sentId))
+ KenLM.destroyPool(poolMap.get(sentId));
+ poolMap.remove(sentId);
+ }
+
+ /**
+ * This function differs from regular transitions because we incorporate the cost of incomplete
+ * left-hand ngrams, as well as including the start- and end-of-sentence markers (if they were
+ * requested when the object was created).
+ *
+ * KenLM already includes the prefix probabilities (of shorter n-grams on the left-hand side), so
+ * there's nothing that needs to be done.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+
+ // KenLMState state = (KenLMState) tailNode.getDPState(getStateIndex());
+
+ // This is unnecessary
+ // acc.add(name, 0.0f);
+
+ // The state is the same since no rule was applied
+ return new KenLMState();
+ }
+
+ /**
+ * KenLM probs already include the prefix probabilities (they are substracted out when merging
+ * states), so this doesn't need to do anything.
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
index 2bdc9ac,0000000..501be62
mode 100644,000000..100644
--- a/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
@@@ -1,210 -1,0 +1,160 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a base class for simple, ASCII line-based grammars that are stored on disk.
+ *
+ * @author Juri Ganitkevitch
+ *
+ */
+public abstract class GrammarReader<R extends Rule> implements Iterable<R>, Iterator<R> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GrammarReader.class);
+
+ protected static String fieldDelimiter;
- protected static String nonTerminalRegEx;
- protected static String nonTerminalCleanRegEx;
+
+ protected static String description;
+
+ protected String fileName;
+ protected LineReader reader;
+ protected String lookAhead;
+ protected int numRulesRead;
+
+
+ // dummy constructor for
+ public GrammarReader() {
+ this.fileName = null;
+ }
+
- public GrammarReader(String fileName) {
++ public GrammarReader(String fileName) throws IOException {
+ this.fileName = fileName;
- }
-
- public void initialize() {
- try {
- this.reader = new LineReader(fileName);
- } catch (IOException e) {
- throw new RuntimeException("Error opening translation model file: " + fileName + "\n"
- + (null != e.getMessage() ? e.getMessage() : "No details available. Sorry."), e);
- }
-
++ this.reader = new LineReader(fileName);
+ LOG.info("Reading grammar from file {}...", fileName);
+ numRulesRead = 0;
+ advanceReader();
+ }
+
+ // the reader is the iterator itself
+ public Iterator<R> iterator() {
+ return this;
+ }
+
+ /** Unsupported Iterator method. */
+ public void remove() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException();
+ }
+
+ public void close() {
+ if (null != this.reader) {
+ try {
+ this.reader.close();
+ } catch (IOException e) {
+ LOG.warn(e.getMessage(), e);
+ LOG.error("Error closing grammar file stream: {}", this.fileName);
+ }
+ this.reader = null;
+ }
+ }
+
+ /**
+ * For correct behavior <code>close</code> must be called on every GrammarReader, however this
+ * code attempts to avoid resource leaks.
+ *
+ * @see org.apache.joshua.util.io.LineReader
+ */
+ @Override
+ protected void finalize() throws Throwable {
+ if (this.reader != null) {
+ LOG.error("Grammar file stream was not closed, this indicates a coding error: {}",
+ this.fileName);
+ }
+
+ this.close();
+ super.finalize();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return lookAhead != null;
+ }
+
+ private void advanceReader() {
+ try {
+ lookAhead = reader.readLine();
+ numRulesRead++;
+ } catch (IOException e) {
+ LOG.error("Error reading grammar from file: {}", fileName);
+ LOG.error(e.getMessage(), e);
+ }
+ if (lookAhead == null && reader != null) {
+ this.close();
+ }
+ }
+
+ /**
+ * Read the next line, and print reader progress.
+ */
+ @Override
+ public R next() {
+ String line = lookAhead;
+
+ int oldProgress = reader.progress();
+ advanceReader();
+
+
+ if (Decoder.VERBOSE >= 1) {
+ int newProgress = (reader != null) ? reader.progress() : 100;
+
+ //TODO: review this code. It is better to print progress based on time gap (like for every 1s or 2sec) than %!
+ if (newProgress > oldProgress) {
+ for (int i = oldProgress + 1; i <= newProgress; i++)
+ if (i == 97) {
+ System.err.print("1");
+ } else if (i == 98) {
+ System.err.print("0");
+ } else if (i == 99) {
+ System.err.print("0");
+ } else if (i == 100) {
+ System.err.println("%");
+ } else if (i % 10 == 0) {
+ System.err.print(String.format("%d", i));
+ System.err.flush();
+ } else if ((i - 1) % 10 == 0)
+ ; // skip at 11 since 10, 20, etc take two digits
+ else {
+ System.err.print(".");
+ System.err.flush();
+ }
+ }
+ }
+ return parseLine(line);
+ }
+
+ protected abstract R parseLine(String line);
-
- // TODO: keep these around or not?
- public abstract String toWords(R rule);
-
- public abstract String toWordsWithoutFeatureScores(R rule);
-
- /**
- * Removes square brackets (and index, if present) from nonterminal id
- * @param tokenID the int ID to clean
- * @return cleaned ID
- */
- public static int cleanNonTerminal(int tokenID) {
- // cleans NT of any markup, e.g., [X,1] may becomes [X], depending
- return Vocabulary.id(cleanNonTerminal(Vocabulary.word(tokenID)));
- }
-
- /**
- * Removes square brackets (and index, if present) from nonterminal id
- * @param token the string ID to clean
- * @return cleaned token
- */
- public static String cleanNonTerminal(String token) {
- // cleans NT of any markup, e.g., [X,1] may becomes [X], depending on nonTerminalCleanRegEx
- return token.replaceAll(nonTerminalCleanRegEx, "");
- }
-
- public static boolean isNonTerminal(final String word) {
- // checks if word matches NT regex
- return word.matches(nonTerminalRegEx);
- }
-
- public String getNonTerminalRegEx() {
- return nonTerminalRegEx;
- }
-
- public String getNonTerminalCleanRegEx() {
- return nonTerminalCleanRegEx;
- }
-
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
index 255324a,0000000..33a2e27
mode 100644,000000..100644
--- a/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
@@@ -1,627 -1,0 +1,627 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class define the interface for Rule.
+ *
+ * All feature scores are interpreted as negative log probabilities, and are therefore negated.
+ * Note that not all features need to be negative log probs, but you should be aware that they
+ * will be negated, so if you want a positive count, it should come in as negative.
+ *
+ * Normally, the feature score in the rule should be *cost* (i.e., -LogP), so that the feature
+ * weight should be positive
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class Rule implements Comparator<Rule>, Comparable<Rule> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Rule.class);
+ private int lhs; // tag of this rule
- private int[] pFrench; // pointer to the RuleCollection, as all the rules under it share the same
++ private int[] source; // pointer to the RuleCollection, as all the rules under it share the same
+ // Source side
+ protected int arity;
+
+ // And a string containing the sparse ones
+ //protected final String sparseFeatureString;
+ protected final Supplier<String> sparseFeatureStringSupplier;
+ private final Supplier<FeatureVector> featuresSupplier;
+
+ /*
+ * a feature function will be fired for this rule only if the owner of the rule matches the owner
+ * of the feature function
+ */
+ private int owner = -1;
+
+ /**
+ * This is the cost computed only from the features present with the grammar rule. This cost is
+ * needed to sort the rules in the grammar for cube pruning, but isn't the full cost of applying
+ * the rule (which will include contextual features that can't be computed until the rule is
+ * applied).
+ */
+ private float estimatedCost = Float.NEGATIVE_INFINITY;
+
+ private float precomputableCost = Float.NEGATIVE_INFINITY;
+
- private int[] english;
++ private int[] target;
+
+ // The alignment string, e.g., 0-0 0-1 1-1 2-1
+ private String alignmentString;
+ private final Supplier<byte[]> alignmentSupplier;
+
+ /**
+ * Constructs a new rule using the provided parameters. Rule id for this rule is
+ * undefined. Note that some of the sparse features may be unlabeled, but they cannot be mapped to
+ * their default names ("tm_OWNER_INDEX") until later, when we know the owner of the rule. This is
+ * not known until the rule is actually added to a grammar in Grammar::addRule().
+ *
+ * Constructor used by other constructors below;
+ *
+ * @param lhs Left-hand side of the rule.
- * @param sourceRhs Source language right-hand side of the rule.
- * @param targetRhs Target language right-hand side of the rule.
++ * @param source Source language right-hand side of the rule.
++ * @param target Target language right-hand side of the rule.
+ * @param sparseFeatures Feature value scores for the rule.
+ * @param arity Number of nonterminals in the source language right-hand side.
+ * @param owner todo
+ */
- public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity, int owner) {
++ public Rule(int lhs, int[] source, int[] target, String sparseFeatures, int arity, int owner) {
+ this.lhs = lhs;
- this.pFrench = sourceRhs;
++ this.source = source;
+ this.arity = arity;
+ this.owner = owner;
- this.english = targetRhs;
++ this.target = target;
+ this.sparseFeatureStringSupplier = Suppliers.memoize(() -> { return sparseFeatures; });
+ this.featuresSupplier = initializeFeatureSupplierFromString();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ /**
+ * Constructor used by PackedGrammar's sortRules()
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param features todo
+ * @param arity todo
+ * @param owner todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, FeatureVector features, int arity, int owner) {
+ this.lhs = lhs;
- this.pFrench = sourceRhs;
++ this.source = sourceRhs;
+ this.arity = arity;
+ this.owner = owner;
- this.english = targetRhs;
++ this.target = targetRhs;
+ this.featuresSupplier = Suppliers.memoize(() -> { return features; });
+ this.sparseFeatureStringSupplier = initializeSparseFeaturesStringSupplier();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ /**
+ * Constructor used for SamtFormatReader and GrammarBuilderWalkerFunction's getRuleWithSpans()
+ * Owner set to -1
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param sparseFeatures todo
+ * @param arity todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity) {
+ this(lhs, sourceRhs, targetRhs, sparseFeatures, arity, -1);
+ }
+
+ /**
+ * Constructor used for addOOVRules(), HieroFormatReader and PhraseRule.
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param sparseFeatures todo
+ * @param arity todo
+ * @param alignment todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity, String alignment) {
+ this(lhs, sourceRhs, targetRhs, sparseFeatures, arity);
+ this.alignmentString = alignment;
+ }
+
+ /**
+ * Constructor (implicitly) used by PackedRule
+ */
+ public Rule() {
+ this.lhs = -1;
+ this.sparseFeatureStringSupplier = initializeSparseFeaturesStringSupplier();
+ this.featuresSupplier = initializeFeatureSupplierFromString();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ // ==========================================================================
+ // Lazy loading Suppliers for alignments, feature vector, and feature strings
+ // ==========================================================================
+
+ private Supplier<byte[]> initializeAlignmentSupplier(){
+ return Suppliers.memoize(() ->{
+ byte[] alignment = null;
+ String alignmentString = getAlignmentString();
+ if (alignmentString != null) {
+ String[] tokens = alignmentString.split("[-\\s]+");
+ alignment = new byte[tokens.length];
+ for (int i = 0; i < tokens.length; i++)
+ alignment[i] = (byte) Short.parseShort(tokens[i]);
+ }
+ return alignment;
+ });
+ }
+
+ /**
+ * If Rule was constructed with sparseFeatures String, we lazily populate the
+ * FeatureSupplier.
+ */
+ private Supplier<FeatureVector> initializeFeatureSupplierFromString(){
+ return Suppliers.memoize(() ->{
+ if (owner != -1) {
+ return new FeatureVector(getFeatureString(), "tm_" + Vocabulary.word(owner) + "_");
+ } else {
+ return new FeatureVector();
+ }
+ });
+ }
+
+ /**
+ * If Rule was constructed with a FeatureVector, we lazily populate the sparseFeaturesStringSupplier.
+ */
+ private Supplier<String> initializeSparseFeaturesStringSupplier() {
+ return Suppliers.memoize(() -> {
+ return getFeatureVector().toString();
+ });
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+
+ public void setEnglish(int[] eng) {
- this.english = eng;
++ this.target = eng;
+ }
+
+ public int[] getEnglish() {
- return this.english;
++ return this.target;
+ }
+
+ /**
+ * Two Rules are equal of they have the same LHS, the same source RHS and the same target
+ * RHS.
+ *
+ * @param o the object to check for equality
+ * @return true if o is the same Rule as this rule, false otherwise
+ */
+ public boolean equals(Object o) {
+ if (!(o instanceof Rule)) {
+ return false;
+ }
+ Rule other = (Rule) o;
+ if (getLHS() != other.getLHS()) {
+ return false;
+ }
+ if (!Arrays.equals(getFrench(), other.getFrench())) {
+ return false;
+ }
- if (!Arrays.equals(english, other.getEnglish())) {
++ if (!Arrays.equals(target, other.getEnglish())) {
+ return false;
+ }
+ return true;
+ }
+
+ public int hashCode() {
+ // I just made this up. If two rules are equal they'll have the
+ // same hashcode. Maybe someone else can do a better job though?
+ int frHash = Arrays.hashCode(getFrench());
- int enHash = Arrays.hashCode(english);
++ int enHash = Arrays.hashCode(target);
+ return frHash ^ enHash ^ getLHS();
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+
+ public void setArity(int arity) {
+ this.arity = arity;
+ }
+
+ public int getArity() {
+ return this.arity;
+ }
+
+ public void setOwner(int owner) {
+ this.owner = owner;
+ }
+
+ public int getOwner() {
+ return this.owner;
+ }
+
+ public void setLHS(int lhs) {
+ this.lhs = lhs;
+ }
+
+ public int getLHS() {
+ return this.lhs;
+ }
+
+ public void setFrench(int[] french) {
- this.pFrench = french;
++ this.source = french;
+ }
+
+ public int[] getFrench() {
- return this.pFrench;
++ return this.source;
+ }
+
+ /**
+ * This function does the work of turning the string version of the sparse features (passed in
+ * when the rule was created) into an actual set of features. This is a bit complicated because we
+ * support intermingled labeled and unlabeled features, where the unlabeled features are mapped to
+ * a default name template of the form "tm_OWNER_INDEX".
+ *
+ * This function returns the dense (phrasal) features discovered when the rule was loaded. Dense
+ * features are the list of unlabeled features that preceded labeled ones. They can also be
+ * specified as labeled features of the form "tm_OWNER_INDEX", but the former format is preferred.
+ *
+ * @return the {@link org.apache.joshua.decoder.ff.FeatureVector} for this rule
+ */
+ public FeatureVector getFeatureVector() {
+ return featuresSupplier.get();
+ }
+
+ /**
+ * This function returns the estimated cost of a rule, which should have been computed when the
+ * grammar was first sorted via a call to Rule::estimateRuleCost(). This function is a getter
+ * only; it will not compute the value if it has not already been set. It is necessary in addition
+ * to estimateRuleCost(models) because sometimes the value needs to be retrieved from contexts
+ * that do not have access to the feature functions.
+ *
+ * This function is called by the rule comparator when sorting the grammar. As such it may be
+ * called many times and any implementation of it should be a cached implementation.
+ *
+ * @return the estimated cost of the rule (a lower bound on the true cost)
+ */
+ public float getEstimatedCost() {
+ return estimatedCost;
+ }
+
+ /**
+ * Precomputable costs is the inner product of the weights found on each grammar rule and the
+ * weight vector. This is slightly different from the estimated rule cost, which can include other
+ * features (such as a language model estimate). This getter and setter should also be cached, and
+ * is basically provided to allow the PhraseModel feature to cache its (expensive) computation for
+ * each rule.
+ *
+ * @return the precomputable cost of each rule
+ */
+ public float getPrecomputableCost() {
+ return precomputableCost;
+ }
+
+ public float getDenseFeature(int k) {
+ return getFeatureVector().getDense(k);
+ }
+
+ public void setPrecomputableCost(float[] phrase_weights, FeatureVector weights) {
+ float cost = 0.0f;
+ FeatureVector features = getFeatureVector();
+ for (int i = 0; i < features.getDenseFeatures().size() && i < phrase_weights.length; i++) {
+ cost += phrase_weights[i] * features.getDense(i);
+ }
+
+ for (String key: features.getSparseFeatures().keySet()) {
+ cost += weights.getSparse(key) * features.getSparse(key);
+ }
+
+ this.precomputableCost = cost;
+ }
+
+ /**
+ * This function estimates the cost of a rule, which is used for sorting the rules for cube
+ * pruning. The estimated cost is basically the set of precomputable features (features listed
+ * along with the rule in the grammar file) along with any other estimates that other features
+ * would like to contribute (e.g., a language model estimate). This cost will be a lower bound on
+ * the rule's actual cost.
+ *
+ * The value of this function is used only for sorting the rules. When the rule is later applied
+ * in context to particular hypernodes, the rule's actual cost is computed.
+ *
+ * @param models the list of models available to the decoder
+ * @return estimated cost of the rule
+ */
+ public float estimateRuleCost(List<FeatureFunction> models) {
+ if (null == models)
+ return 0.0f;
+
+ if (this.estimatedCost <= Float.NEGATIVE_INFINITY) {
+ this.estimatedCost = 0.0f; // weights.innerProduct(computeFeatures());
+
+ LOG.debug("estimateCost({} ;; {})", getFrenchWords(), getEnglishWords());
+ for (FeatureFunction ff : models) {
+ float val = ff.estimateCost(this, null);
+ LOG.debug(" FEATURE {} -> {}", ff.getName(), val);
+ this.estimatedCost += val;
+ }
+ }
+
+ return estimatedCost;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" ||| ");
+ sb.append(getFrenchWords());
+ sb.append(" ||| ");
+ sb.append(getEnglishWords());
+ sb.append(" |||");
+ sb.append(" " + getFeatureVector());
+ sb.append(String.format(" ||| est=%.3f", getEstimatedCost()));
+ sb.append(String.format(" pre=%.3f", getPrecomputableCost()));
+ return sb.toString();
+ }
+
+ /**
+ * Returns a version of the rule suitable for reading in from a text file.
+ *
+ * @return string version of the rule
+ */
+ public String textFormat() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" |||");
+
+ int nt = 1;
+ for (int i = 0; i < getFrench().length; i++) {
+ if (getFrench()[i] < 0)
+ sb.append(" " + Vocabulary.word(getFrench()[i]).replaceFirst("\\]", String.format(",%d]", nt++)));
+ else
+ sb.append(" " + Vocabulary.word(getFrench()[i]));
+ }
+ sb.append(" |||");
+ nt = 1;
+ for (int i = 0; i < getEnglish().length; i++) {
+ if (getEnglish()[i] < 0)
+ sb.append(" " + Vocabulary.word(getEnglish()[i]).replaceFirst("\\]", String.format(",%d]", nt++)));
+ else
+ sb.append(" " + Vocabulary.word(getEnglish()[i]));
+ }
+ sb.append(" |||");
+ sb.append(" " + getFeatureString());
+ if (getAlignmentString() != null)
+ sb.append(" ||| " + getAlignmentString());
+ return sb.toString();
+ }
+
+ public String getFeatureString() {
+ return sparseFeatureStringSupplier.get();
+ }
+
+ /**
+ * Returns an alignment as a sequence of integers. The integers at positions i and i+1 are paired,
+ * with position i indexing the source and i+1 the target.
+ *
+ * @return a byte[] from the {@link com.google.common.base.Supplier}
+ */
+ public byte[] getAlignment() {
+ return this.alignmentSupplier.get();
+ }
+
+ public String getAlignmentString() {
+ return this.alignmentString;
+ }
+
+ /**
+ * The nonterminals on the English side are pointers to the source side nonterminals (-1 and -2),
+ * rather than being directly encoded. These number indicate the correspondence between the
+ * nonterminals on each side, introducing a level of indirection however when we want to resolve
+ * them. So to get the ID, we need to look up the corresponding source side ID.
+ *
+ * @return The string of English words
+ */
+ public String getEnglishWords() {
+ int[] foreignNTs = getForeignNonTerminals();
+
+ StringBuilder sb = new StringBuilder();
+ for (Integer index : getEnglish()) {
+ if (index >= 0)
+ sb.append(Vocabulary.word(index) + " ");
+ else
+ sb.append(Vocabulary.word(foreignNTs[-index - 1]).replace("]",
+ String.format(",%d] ", Math.abs(index))));
+ }
+
+ return sb.toString().trim();
+ }
+
+ public boolean isTerminal() {
+ for (int i = 0; i < getEnglish().length; i++)
+ if (getEnglish()[i] < 0)
+ return false;
+
+ return true;
+ }
+
+ /**
+ * Return the French (source) nonterminals as list of Strings
+ *
+ * @return a list of strings
+ */
+ public int[] getForeignNonTerminals() {
+ int[] nts = new int[getArity()];
+ int index = 0;
+ for (int id : getFrench())
+ if (id < 0)
+ nts[index++] = -id;
+ return nts;
+ }
+
+ /**
+ * Returns an array of size getArity() containing the source indeces of non terminals.
+ *
+ * @return an array of size getArity() containing the source indeces of non terminals
+ */
+ public int[] getNonTerminalSourcePositions() {
+ int[] nonTerminalPositions = new int[getArity()];
+ int ntPos = 0;
+ for (int sourceIdx = 0; sourceIdx < getFrench().length; sourceIdx++) {
+ if (getFrench()[sourceIdx] < 0)
+ nonTerminalPositions[ntPos++] = sourceIdx;
+ }
+ return nonTerminalPositions;
+ }
+
+ /**
+ * Parses the Alignment byte[] into a Map from target to (possibly a list of) source positions.
+ * Used by the WordAlignmentExtractor.
+ *
+ * @return a {@link java.util.Map} of alignments
+ */
+ public Map<Integer, List<Integer>> getAlignmentMap() {
+ byte[] alignmentArray = getAlignment();
+ Map<Integer, List<Integer>> alignmentMap = new HashMap<Integer, List<Integer>>();
+ if (alignmentArray != null) {
+ for (int alignmentIdx = 0; alignmentIdx < alignmentArray.length; alignmentIdx += 2 ) {
+ int s = alignmentArray[alignmentIdx];
+ int t = alignmentArray[alignmentIdx + 1];
+ List<Integer> values = alignmentMap.get(t);
+ if (values == null)
+ alignmentMap.put(t, values = new ArrayList<Integer>());
+ values.add(s);
+ }
+ }
+ return alignmentMap;
+ }
+
+ /**
+ * Return the English (target) nonterminals as list of Strings
+ *
+ * @return list of strings
+ */
+ public int[] getEnglishNonTerminals() {
+ int[] nts = new int[getArity()];
+ int[] foreignNTs = getForeignNonTerminals();
+ int index = 0;
+
+ for (int i : getEnglish()) {
+ if (i < 0)
+ nts[index++] = foreignNTs[Math.abs(getEnglish()[i]) - 1];
+ }
+
+ return nts;
+ }
+
+ private int[] getNormalizedEnglishNonterminalIndices() {
+ int[] result = new int[getArity()];
+
+ int ntIndex = 0;
+ for (Integer index : getEnglish()) {
+ if (index < 0)
+ result[ntIndex++] = -index - 1;
+ }
+
+ return result;
+ }
+
+ public boolean isInverting() {
+ int[] normalizedEnglishNonTerminalIndices = getNormalizedEnglishNonterminalIndices();
+ if (normalizedEnglishNonTerminalIndices.length == 2) {
+ if (normalizedEnglishNonTerminalIndices[0] == 1) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public String getFrenchWords() {
+ return Vocabulary.getWords(getFrench());
+ }
+
+ public static final String NT_REGEX = "\\[[^\\]]+?\\]";
+
+ private Pattern getPattern() {
+ String source = getFrenchWords();
+ String pattern = Pattern.quote(source);
+ pattern = pattern.replaceAll(NT_REGEX, "\\\\E.+\\\\Q");
+ pattern = pattern.replaceAll("\\\\Q\\\\E", "");
+ pattern = "(?:^|\\s)" + pattern + "(?:$|\\s)";
+ return Pattern.compile(pattern);
+ }
+
+ /**
+ * Matches the string representation of the rule's source side against a sentence
+ *
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return true if there is a match
+ */
+ public boolean matches(Sentence sentence) {
+ boolean match = getPattern().matcher(sentence.fullSource()).find();
+ // System.err.println(String.format("match(%s,%s) = %s", Pattern.quote(getFrenchWords()),
+ // sentence.annotatedSource(), match));
+ return match;
+ }
+
+ /**
+ * This comparator is used for sorting the rules during cube pruning. An estimate of the cost
+ * of each rule is computed and used to sort.
+ */
+ public static Comparator<Rule> EstimatedCostComparator = new Comparator<Rule>() {
+ public int compare(Rule rule1, Rule rule2) {
+ float cost1 = rule1.getEstimatedCost();
+ float cost2 = rule2.getEstimatedCost();
+ return Float.compare(cost2, cost1);
+ }
+ };
+
+ public int compare(Rule rule1, Rule rule2) {
+ return EstimatedCostComparator.compare(rule1, rule2);
+ }
+
+ public int compareTo(Rule other) {
+ return EstimatedCostComparator.compare(this, other);
+ }
+
+ public String getRuleString() {
+ return String.format("%s -> %s ||| %s", Vocabulary.word(getLHS()), getFrenchWords(), getEnglishWords());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
index 04a206a,0000000..d80638d
mode 100644,000000..100644
--- a/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
@@@ -1,122 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.format;
+
++import java.io.IOException;
++
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.GrammarReader;
+import org.apache.joshua.decoder.ff.tm.Rule;
++import org.apache.joshua.util.FormatUtils;
+
+/**
+ * This class implements reading files in the format defined by David Chiang for Hiero.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class HieroFormatReader extends GrammarReader<Rule> {
+
+ static {
+ fieldDelimiter = "\\s\\|{3}\\s";
- nonTerminalRegEx = "^\\[[^\\s]+\\,[0-9]*\\]$";
- nonTerminalCleanRegEx = ",[0-9\\s]+";
- // nonTerminalRegEx = "^\\[[A-Z]+\\,[0-9]*\\]$";
- // nonTerminalCleanRegEx = "[\\[\\]\\,0-9\\s]+";
+ description = "Original Hiero format";
+ }
+
+ public HieroFormatReader() {
+ super();
+ }
+
- public HieroFormatReader(String grammarFile) {
++ public HieroFormatReader(String grammarFile) throws IOException {
+ super(grammarFile);
+ }
+
+ @Override
+ public Rule parseLine(String line) {
+ String[] fields = line.split(fieldDelimiter);
+ if (fields.length < 3) {
+ throw new RuntimeException(String.format("Rule '%s' does not have four fields", line));
+ }
+
- int lhs = Vocabulary.id(cleanNonTerminal(fields[0]));
++ int lhs = Vocabulary.id(fields[0]);
+
++ /**
++ * On the foreign side, we map nonterminals to negative IDs, and terminals to positive IDs.
++ */
+ int arity = 0;
- // foreign side
- String[] foreignWords = fields[1].split("\\s+");
- int[] french = new int[foreignWords.length];
- for (int i = 0; i < foreignWords.length; i++) {
- french[i] = Vocabulary.id(foreignWords[i]);
- if (Vocabulary.nt(french[i])) {
++ String[] sourceWords = fields[1].split("\\s+");
++ int[] sourceIDs = new int[sourceWords.length];
++ for (int i = 0; i < sourceWords.length; i++) {
++ sourceIDs[i] = Vocabulary.id(sourceWords[i]);
++ if (FormatUtils.isNonterminal(sourceWords[i])) {
++ sourceIDs[i] = Vocabulary.id(FormatUtils.stripNonTerminalIndex(sourceWords[i]));
+ arity++;
- french[i] = cleanNonTerminal(french[i]);
++
++ // TODO: the arity here (after incrementing) should match the rule index. Should
++ // check that arity == FormatUtils.getNonterminalIndex(foreignWords[i]), throw runtime
++ // error if not
+ }
+ }
+
- // English side
- String[] englishWords = fields[2].split("\\s+");
- int[] english = new int[englishWords.length];
- for (int i = 0; i < englishWords.length; i++) {
- english[i] = Vocabulary.id(englishWords[i]);
- if (Vocabulary.nt(english[i])) {
- english[i] = -Vocabulary.getTargetNonterminalIndex(english[i]);
++ /**
++ * The English side maps terminal symbols to positive IDs. Nonterminal symbols are linked
++ * to the index of the source-side nonterminal they are linked to. So for a rule
++ *
++ * [X] ||| [X,1] [X,2] [X,3] ||| [X,2] [X,1] [X,3] ||| ...
++ *
++ * the English side nonterminals will be -2, -1, -3. This assumes that the source side of
++ * the rule is always listed monotonically.
++ */
++ String[] targetWords = fields[2].split("\\s+");
++ int[] targetIDs = new int[targetWords.length];
++ for (int i = 0; i < targetWords.length; i++) {
++ targetIDs[i] = Vocabulary.id(targetWords[i]);
++ if (FormatUtils.isNonterminal(targetWords[i])) {
++ targetIDs[i] = -FormatUtils.getNonterminalIndex(targetWords[i]);
+ }
+ }
+
+ String sparse_features = (fields.length > 3 ? fields[3] : "");
+ String alignment = (fields.length > 4) ? fields[4] : null;
+
- return new Rule(lhs, french, english, sparse_features, arity, alignment);
- }
-
- @Override
- public String toWords(Rule rule) {
- StringBuffer sb = new StringBuffer("");
- sb.append(Vocabulary.word(rule.getLHS()));
- sb.append(" ||| ");
- sb.append(Vocabulary.getWords(rule.getFrench()));
- sb.append(" ||| ");
- sb.append(Vocabulary.getWords(rule.getEnglish()));
- sb.append(" |||");
- sb.append(" " + rule.getFeatureVector());
-
- return sb.toString();
++ return new Rule(lhs, sourceIDs, targetIDs, sparse_features, arity, alignment);
+ }
+
- @Override
- public String toWordsWithoutFeatureScores(Rule rule) {
- StringBuffer sb = new StringBuffer();
- sb.append(rule.getLHS());
- sb.append(" ||| ");
- sb.append(Vocabulary.getWords(rule.getFrench()));
- sb.append(" ||| ");
- sb.append(Vocabulary.getWords(rule.getEnglish()));
- sb.append(" |||");
-
- return sb.toString();
- }
-
-
+ public static String getFieldDelimiter() {
+ return fieldDelimiter;
+ }
+
+ public static boolean isNonTerminal(final String word) {
- return GrammarReader.isNonTerminal(word);
++ return FormatUtils.isNonterminal(word);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/abb8c518/src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java
index 0000000,0000000..fcab917
new file mode 100644
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java
@@@ -1,0 -1,0 +1,106 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++package org.apache.joshua.decoder.ff.tm.format;
++
++import java.io.IOException;
++
++import org.apache.joshua.corpus.Vocabulary;
++import org.apache.joshua.decoder.ff.tm.Rule;
++import org.apache.joshua.util.io.LineReader;
++
++/***
++ * This class reads in the Moses phrase table format, with support for the source and target side,
++ * list of features, and word alignments. It works by
++ *
++ * - casting the phrase-based rules to left-branching hierarchical rules and passing them on \
++ * to its parent class, {@HieroFormatReader}.
++ * - converting the probabilities to -log probabilities
++ *
++ * There is also a tool to convert the grammars directly, so that they can be suitably packed. Usage:
++ *
++ * <pre>
++ * cat PHRASE_TABLE | java -cp $JOSHUA/class joshua.decoder.ff.tm.format.MosesFormatReader > grammar
++ * </pre>
++ *
++ * @author Matt Post <po...@cs.jhu.edu>
++ *
++ */
++
++public class MosesFormatReader extends HieroFormatReader {
++
++ public MosesFormatReader(String grammarFile) throws IOException {
++ super(grammarFile);
++ Vocabulary.id("[X]");
++ }
++
++ public MosesFormatReader() {
++ super();
++ Vocabulary.id("[X]");
++ }
++
++ /**
++ * When dealing with Moses format, this munges a Moses-style phrase table into a grammar.
++ *
++ * mots francaises ||| French words ||| 1 2 3 ||| 0-1 1-0
++ *
++ * becomes
++ *
++ * [X] ||| [X,1] mots francaises ||| [X,1] French words ||| 1 2 3 ||| 0-1 1-0
++ *
++ * For thrax-extracted phrasal grammars, it transforms
++ *
++ * [X] ||| mots francaises ||| French words ||| 1 2 3 ||| 0-1 1-0
++ *
++ * into
++ *
++ * [X] ||| [X,1] mots francaises ||| [X,1] French words ||| 1 2 3 ||| 0-1 1-0
++ */
++ @Override
++ public Rule parseLine(String line) {
++ String[] fields = line.split(fieldDelimiter);
++
++ StringBuffer hieroLine = new StringBuffer();
++ hieroLine.append("[X] ||| [X,1] " + fields[0] + " ||| [X,1] " + fields[1] + " |||");
++
++ String mosesFeatureString = fields[2];
++ for (String value: mosesFeatureString.split(" ")) {
++ float f = Float.parseFloat(value);
++ hieroLine.append(String.format(" %f", f <= 0.0 ? -100 : -Math.log(f)));
++ }
++
++ // alignments
++ if (fields.length >= 4)
++ hieroLine.append(" ||| " + fields[3]);
++
++ return super.parseLine(hieroLine.toString());
++ }
++
++ /**
++ * Converts a Moses phrase table to a Joshua grammar.
++ *
++ * @param args
++ */
++ public static void main(String[] args) {
++ MosesFormatReader reader = new MosesFormatReader();
++ for (String line: new LineReader(System.in)) {
++ Rule rule = reader.parseLine(line);
++ System.out.println(rule.textFormat());
++ }
++ }
++}