You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/23 18:46:10 UTC
[59/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
index 0000000,097ce59..f319e40
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
@@@ -1,0 -1,813 +1,813 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder;
+
+ import static org.apache.joshua.decoder.ff.FeatureVector.DENSE_FEATURE_NAMES;
++import static org.apache.joshua.decoder.ff.tm.OwnerMap.getOwner;
+
+ import java.io.BufferedWriter;
+ import java.io.File;
+ import java.io.IOException;
+ import java.io.FileNotFoundException;
+ import java.lang.reflect.Constructor;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
++import java.util.Set;
+ import java.util.concurrent.ArrayBlockingQueue;
+ import java.util.concurrent.BlockingQueue;
+
+ import com.google.common.base.Strings;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.PhraseModel;
+ import org.apache.joshua.decoder.ff.StatefulFF;
+ import org.apache.joshua.decoder.ff.lm.LanguageModelFF;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
+ import org.apache.joshua.decoder.io.TranslationRequestStream;
+ import org.apache.joshua.decoder.phrase.PhraseTable;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.util.FileUtility;
+ import org.apache.joshua.util.FormatUtils;
+ import org.apache.joshua.util.Regex;
+ import org.apache.joshua.util.io.LineReader;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This class handles decoder initialization and the complication introduced by multithreading.
+ *
+ * After initialization, the main entry point to the Decoder object is
+ * decodeAll(TranslationRequest), which returns a set of Translation objects wrapped in an iterable
+ * Translations object. It is important that we support multithreading both (a) across the sentences
+ * within a request and (b) across requests, in a round-robin fashion. This is done by maintaining a
+ * fixed sized concurrent thread pool. When a new request comes in, a RequestParallelizer thread is
+ * launched. This object iterates over the request's sentences, obtaining a thread from the
+ * thread pool, and using that thread to decode the sentence. If a decoding thread is not available,
+ * it will block until one is in a fair (FIFO) manner. RequestParallelizer thereby permits intra-request
+ * parallelization by separating out reading the input stream from processing the translated sentences,
+ * but also ensures that round-robin parallelization occurs, since RequestParallelizer uses the
+ * thread pool before translating each request.
+ *
+ * A decoding thread is handled by DecoderThread and launched from DecoderThreadRunner. The purpose
+ * of the runner is to record where to place the translated sentence when it is done (i.e., which
+ * Translations object). Translations itself is an iterator whose next() call blocks until the next
+ * translation is available.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Lane Schwartz dowobeha@users.sourceforge.net
+ */
+ public class Decoder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Decoder.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+
+ public JoshuaConfiguration getJoshuaConfiguration() {
+ return joshuaConfiguration;
+ }
+
+ /*
+ * Many of these objects themselves are global objects. We pass them in when constructing other
+ * objects, so that they all share pointers to the same object. This is good because it reduces
+ * overhead, but it can be problematic because of unseen dependencies (for example, in the
+ * Vocabulary shared by language model, translation grammar, etc).
+ */
+ private List<Grammar> grammars;
+ private ArrayList<FeatureFunction> featureFunctions;
+ private Grammar customPhraseTable;
+
+ /* The feature weights. */
+ public static FeatureVector weights;
+
+ public static int VERBOSE = 1;
+
+ private BlockingQueue<DecoderThread> threadPool = null;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ /**
+ * Constructor method that creates a new decoder using the specified configuration file.
+ *
+ * @param joshuaConfiguration a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @param configFile name of configuration file.
+ */
+ public Decoder(JoshuaConfiguration joshuaConfiguration, String configFile) {
+ this(joshuaConfiguration);
+ this.initialize(configFile);
+ }
+
+ /**
+ * Factory method that creates a new decoder using the specified configuration file.
+ *
+ * @param configFile Name of configuration file.
+ * @return a configured {@link org.apache.joshua.decoder.Decoder}
+ */
+ public static Decoder createDecoder(String configFile) {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ return new Decoder(joshuaConfiguration, configFile);
+ }
+
+ /**
+ * Constructs an uninitialized decoder for use in testing.
+ * <p>
+ * This method is private because it should only ever be called by the
+ * {@link #getUninitalizedDecoder()} method to provide an uninitialized decoder for use in
+ * testing.
+ */
+ private Decoder(JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.grammars = new ArrayList<Grammar>();
+ this.threadPool = new ArrayBlockingQueue<DecoderThread>(
+ this.joshuaConfiguration.num_parallel_decoders, true);
+ this.customPhraseTable = null;
+ }
+
+ /**
+ * Gets an uninitialized decoder for use in testing.
+ * <p>
+ * This method is called by unit tests or any outside packages (e.g., MERT) relying on the
+ * decoder.
+ * @param joshuaConfiguration a {@link org.apache.joshua.decoder.JoshuaConfiguration} object
+ * @return an uninitialized decoder for use in testing
+ */
+ static public Decoder getUninitalizedDecoder(JoshuaConfiguration joshuaConfiguration) {
+ return new Decoder(joshuaConfiguration);
+ }
+
+ // ===============================================================
+ // Public Methods
+ // ===============================================================
+
+ /**
+ * This class is responsible for getting sentences from the TranslationRequest and procuring a
+ * DecoderThreadRunner to translate it. Each call to decodeAll(TranslationRequest) launches a
+ * thread that will read the request's sentences, obtain a DecoderThread to translate them, and
+ * then place the Translation in the appropriate place.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ *
+ */
+ private class RequestParallelizer extends Thread {
+ /* Source of sentences to translate. */
+ private final TranslationRequestStream request;
+
+ /* Where to put translated sentences. */
+ private final Translations response;
+
+ RequestParallelizer(TranslationRequestStream request, Translations response) {
+ this.request = request;
+ this.response = response;
+ }
+
+ @Override
+ public void run() {
+ /*
+ * Repeatedly get an input sentence, wait for a DecoderThread, and then start a new thread to
+ * translate the sentence. We start a new thread (via DecoderRunnerThread) as opposed to
+ * blocking, so that the RequestHandler can go on to the next sentence in this request, which
+ * allows parallelization across the sentences of the request.
+ */
+ for (;;) {
+ Sentence sentence = request.next();
+
+ if (sentence == null) {
+ response.finish();
+ break;
+ }
+
+ // This will block until a DecoderThread becomes available.
+ DecoderThread thread = Decoder.this.getThread();
+ new DecoderThreadRunner(thread, sentence, response).start();
+ }
+ }
+
+ /**
+ * Strips the nonterminals from the lefthand side of the rule.
+ *
+ * @param rule
+ * @return
+ */
+ private String formatRule(Rule rule) {
+ String ruleString = "";
+ boolean first = true;
+ for (int word: rule.getFrench()) {
+ if (!first)
+ ruleString += " " + Vocabulary.word(word);
+ first = false;
+ }
+
+ ruleString += " |||"; // space will get added with first English word
+ first = true;
+ for (int word: rule.getEnglish()) {
+ if (!first)
+ ruleString += " " + Vocabulary.word(word);
+ first = false;
+ }
+
+ // strip of the leading space
+ return ruleString.substring(1);
+ }
+ }
+
+ /**
+ * Retrieve a thread from the thread pool, blocking until one is available. The blocking occurs in
+ * a fair fashion (i.e,. FIFO across requests).
+ *
+ * @return a thread that can be used for decoding.
+ */
+ public DecoderThread getThread() {
+ try {
+ return threadPool.take();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * This class handles running a DecoderThread (which takes care of the actual translation of an
+ * input Sentence, returning a Translation object when its done). This is done in a thread so as
+ * not to tie up the RequestHandler that launched it, freeing it to go on to the next sentence in
+ * the TranslationRequest, in turn permitting parallelization across the sentences of a request.
+ *
+ * When the decoder thread is finshed, the Translation object is placed in the correct place in
+ * the corresponding Translations object that was returned to the caller of
+ * Decoder.decodeAll(TranslationRequest).
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+ private class DecoderThreadRunner extends Thread {
+
+ private final DecoderThread decoderThread;
+ private final Sentence sentence;
+ private final Translations translations;
+
+ DecoderThreadRunner(DecoderThread thread, Sentence sentence, Translations translations) {
+ this.decoderThread = thread;
+ this.sentence = sentence;
+ this.translations = translations;
+ }
+
+ @Override
+ public void run() {
+ /*
+ * Process any found metadata.
+ */
+
+ /*
+ * Use the thread to translate the sentence. Then record the translation with the
+ * corresponding Translations object, and return the thread to the pool.
+ */
+ try {
+ Translation translation = decoderThread.translate(this.sentence);
+ translations.record(translation);
+
+ /*
+ * This is crucial! It's what makes the thread available for the next sentence to be
+ * translated.
+ */
+ threadPool.put(decoderThread);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format(
+ "Input %d: FATAL UNCAUGHT EXCEPTION: %s", sentence.id(), e.getMessage()), e);
+ // translations.record(new Translation(sentence, null, featureFunctions, joshuaConfiguration));
+ }
+ }
+ }
+
+ /**
+ * This function is the main entry point into the decoder. It translates all the sentences in a
+ * (possibly boundless) set of input sentences. Each request launches its own thread to read the
+ * sentences of the request.
+ *
+ * @param request the populated {@link org.apache.joshua.decoder.io.TranslationRequestStream}
+ * @throws IOException if there is an error with the input stream or writing the output
+ * @return an iterable, asynchronously-filled list of Translations
+ */
+ public Translations decodeAll(TranslationRequestStream request) throws IOException {
+ Translations translations = new Translations(request);
+
+ /* Start a thread to handle requests on the input stream */
+ new RequestParallelizer(request, translations).start();
+
+ return translations;
+ }
+
+
+ /**
+ * We can also just decode a single sentence.
+ *
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return the sentence {@link org.apache.joshua.decoder.Translation}
+ */
+ public Translation decode(Sentence sentence) {
+ // Get a thread.
+
+ try {
+ DecoderThread thread = threadPool.take();
+ Translation translation = thread.translate(sentence);
+ threadPool.put(thread);
+
+ return translation;
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * Clean shutdown of Decoder, resetting all
+ * static variables, such that any other instance of Decoder
+ * afterwards gets a fresh start.
+ */
+ public void cleanUp() {
+ // shut down DecoderThreads
+ for (DecoderThread thread : threadPool) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ resetGlobalState();
+ }
+
+ public static void resetGlobalState() {
+ // clear/reset static variables
++ OwnerMap.clear();
+ DENSE_FEATURE_NAMES.clear();
+ Vocabulary.clear();
+ Vocabulary.unregisterLanguageModels();
+ LanguageModelFF.resetLmIndex();
+ StatefulFF.resetGlobalStateIndex();
+ }
+
+ public static void writeConfigFile(double[] newWeights, String template, String outputFile,
+ String newDiscriminativeModel) {
+ try {
+ int columnID = 0;
+
+ BufferedWriter writer = FileUtility.getWriteFileStream(outputFile);
+ LineReader reader = new LineReader(template);
+ try {
+ for (String line : reader) {
+ line = line.trim();
+ if (Regex.commentOrEmptyLine.matches(line) || line.indexOf("=") != -1) {
+ // comment, empty line, or parameter lines: just copy
+ writer.write(line);
+ writer.newLine();
+
+ } else { // models: replace the weight
+ String[] fds = Regex.spaces.split(line);
+ StringBuffer newSent = new StringBuffer();
+ if (!Regex.floatingNumber.matches(fds[fds.length - 1])) {
+ throw new IllegalArgumentException("last field is not a number; the field is: "
+ + fds[fds.length - 1]);
+ }
+
+ if (newDiscriminativeModel != null && "discriminative".equals(fds[0])) {
+ newSent.append(fds[0]).append(' ');
+ newSent.append(newDiscriminativeModel).append(' ');// change the
+ // file name
+ for (int i = 2; i < fds.length - 1; i++) {
+ newSent.append(fds[i]).append(' ');
+ }
+ } else {// regular
+ for (int i = 0; i < fds.length - 1; i++) {
+ newSent.append(fds[i]).append(' ');
+ }
+ }
+ if (newWeights != null)
+ newSent.append(newWeights[columnID++]);// change the weight
+ else
+ newSent.append(fds[fds.length - 1]);// do not change
+
+ writer.write(newSent.toString());
+ writer.newLine();
+ }
+ }
+ } finally {
+ reader.close();
+ writer.close();
+ }
+
+ if (newWeights != null && columnID != newWeights.length) {
+ throw new IllegalArgumentException("number of models does not match number of weights");
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // ===============================================================
+ // Initialization Methods
+ // ===============================================================
+
+ /**
+ * Moses requires the pattern .*_.* for sparse features, and prohibits underscores in dense features.
+ * This conforms to that pattern. We assume non-conforming dense features start with tm_ or lm_,
+ * and the only sparse feature that needs converting is OOVPenalty.
+ *
+ * @param feature
+ * @return the feature in Moses format
+ */
+ private String mosesize(String feature) {
+ if (joshuaConfiguration.moses) {
+ if (feature.startsWith("tm_") || feature.startsWith("lm_"))
+ return feature.replace("_", "-");
+ }
+
+ return feature;
+ }
+
+ /**
+ * Initialize all parts of the JoshuaDecoder.
+ *
+ * @param configFile File containing configuration options
+ * @return An initialized decoder
+ */
+ public Decoder initialize(String configFile) {
+ try {
+
+ long pre_load_time = System.currentTimeMillis();
+
+ /* Weights can be listed in a separate file (denoted by parameter "weights-file") or directly
+ * in the Joshua config file. Config file values take precedent.
+ */
+ this.readWeights(joshuaConfiguration.weights_file);
+
+
+ /* Add command-line-passed weights to the weights array for processing below */
+ if (!Strings.isNullOrEmpty(joshuaConfiguration.weight_overwrite)) {
+ String[] tokens = joshuaConfiguration.weight_overwrite.split("\\s+");
+ for (int i = 0; i < tokens.length; i += 2) {
+ String feature = tokens[i];
+ float value = Float.parseFloat(tokens[i+1]);
+
+ if (joshuaConfiguration.moses)
+ feature = demoses(feature);
+
+ joshuaConfiguration.weights.add(String.format("%s %s", feature, tokens[i+1]));
+ LOG.info("COMMAND LINE WEIGHT: {} -> {}", feature, value);
+ }
+ }
+
+ /* Read the weights found in the config file */
+ for (String pairStr: joshuaConfiguration.weights) {
+ String pair[] = pairStr.split("\\s+");
+
+ /* Sanity check for old-style unsupported feature invocations. */
+ if (pair.length != 2) {
+ StringBuilder errMsg = new StringBuilder();
+ errMsg.append("FATAL: Invalid feature weight line found in config file.\n");
+ errMsg.append(String.format("The line was '%s'\n", pairStr));
+ errMsg.append("You might be using an old version of the config file that is no longer supported\n");
+ errMsg.append("Check joshua-decoder.org or email joshua_support@googlegroups.com for help\n");
+ errMsg.append("Code = " + 17);
+ throw new RuntimeException(errMsg.toString());
+ }
+
+ weights.set(pair[0], Float.parseFloat(pair[1]));
+ }
+
+ LOG.info("Read {} weights ({} of them dense)", weights.size(), DENSE_FEATURE_NAMES.size());
+
+ // Do this before loading the grammars and the LM.
+ this.featureFunctions = new ArrayList<FeatureFunction>();
+
+ // Initialize and load grammars. This must happen first, since the vocab gets defined by
+ // the packed grammar (if any)
+ this.initializeTranslationGrammars();
+ LOG.info("Grammar loading took: {} seconds.",
+ (System.currentTimeMillis() - pre_load_time) / 1000);
+
+ // Initialize the features: requires that LM model has been initialized.
+ this.initializeFeatureFunctions();
+
+ // This is mostly for compatibility with the Moses tuning script
+ if (joshuaConfiguration.show_weights_and_quit) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ String name = DENSE_FEATURE_NAMES.get(i);
+ if (joshuaConfiguration.moses)
+ System.out.println(String.format("%s= %.5f", mosesize(name), weights.getDense(i)));
+ else
+ System.out.println(String.format("%s %.5f", name, weights.getDense(i)));
+ }
+ System.exit(0);
+ }
+
+ // Sort the TM grammars (needed to do cube pruning)
+ if (joshuaConfiguration.amortized_sorting) {
+ LOG.info("Grammar sorting happening lazily on-demand.");
+ } else {
+ long pre_sort_time = System.currentTimeMillis();
+ for (Grammar grammar : this.grammars) {
+ grammar.sortGrammar(this.featureFunctions);
+ }
+ LOG.info("Grammar sorting took {} seconds.",
+ (System.currentTimeMillis() - pre_sort_time) / 1000);
+ }
+
+ // Create the threads
+ for (int i = 0; i < joshuaConfiguration.num_parallel_decoders; i++) {
+ this.threadPool.put(new DecoderThread(this.grammars, Decoder.weights,
+ this.featureFunctions, joshuaConfiguration));
+ }
+ } catch (IOException | InterruptedException e) {
+ LOG.warn(e.getMessage(), e);
+ }
+
+ return this;
+ }
+
+ /**
+ * Initializes translation grammars Retained for backward compatibility
+ *
+ * @param ownersSeen Records which PhraseModelFF's have been instantiated (one is needed for each
+ * owner)
+ * @throws IOException
+ */
+ private void initializeTranslationGrammars() throws IOException {
+
+ if (joshuaConfiguration.tms.size() > 0) {
+
+ // collect packedGrammars to check if they use a shared vocabulary
+ final List<PackedGrammar> packed_grammars = new ArrayList<>();
+
+ // tm = {thrax/hiero,packed,samt,moses} OWNER LIMIT FILE
+ for (String tmLine : joshuaConfiguration.tms) {
+
+ String type = tmLine.substring(0, tmLine.indexOf(' '));
+ String[] args = tmLine.substring(tmLine.indexOf(' ')).trim().split("\\s+");
+ HashMap<String, String> parsedArgs = FeatureFunction.parseArgs(args);
+
+ String owner = parsedArgs.get("owner");
+ int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
+ String path = parsedArgs.get("path");
+
+ Grammar grammar = null;
+ if (! type.equals("moses") && ! type.equals("phrase")) {
+ if (new File(path).isDirectory()) {
+ try {
+ PackedGrammar packed_grammar = new PackedGrammar(path, span_limit, owner, type, joshuaConfiguration);
+ packed_grammars.add(packed_grammar);
+ grammar = packed_grammar;
+ } catch (FileNotFoundException e) {
+ String msg = String.format("Couldn't load packed grammar from '%s'", path)
+ + "Perhaps it doesn't exist, or it may be an old packed file format.";
+ throw new RuntimeException(msg);
+ }
+ } else {
+ // thrax, hiero, samt
+ grammar = new MemoryBasedBatchGrammar(type, path, owner,
+ joshuaConfiguration.default_non_terminal, span_limit, joshuaConfiguration);
+ }
+
+ } else {
+
- int maxSourceLen = parsedArgs.containsKey("max-source-len")
- ? Integer.parseInt(parsedArgs.get("max-source-len"))
- : -1;
-
+ joshuaConfiguration.search_algorithm = "stack";
+ grammar = new PhraseTable(path, owner, type, joshuaConfiguration);
+ }
+
+ this.grammars.add(grammar);
+ }
+
+ checkSharedVocabularyChecksumsForPackedGrammars(packed_grammars);
+
+ } else {
+ LOG.warn("no grammars supplied! Supplying dummy glue grammar.");
- MemoryBasedBatchGrammar glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration);
- glueGrammar.setSpanLimit(-1);
++ MemoryBasedBatchGrammar glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration, -1);
+ glueGrammar.addGlueRules(featureFunctions);
+ this.grammars.add(glueGrammar);
+ }
+
+ /* Add the grammar for custom entries */
+ if (joshuaConfiguration.search_algorithm.equals("stack"))
+ this.customPhraseTable = new PhraseTable(null, "custom", "phrase", joshuaConfiguration);
+ else
- this.customPhraseTable = new MemoryBasedBatchGrammar("custom", joshuaConfiguration);
++ this.customPhraseTable = new MemoryBasedBatchGrammar("custom", joshuaConfiguration, 20);
+ this.grammars.add(this.customPhraseTable);
+
+ /* Create an epsilon-deleting grammar */
+ if (joshuaConfiguration.lattice_decoding) {
+ LOG.info("Creating an epsilon-deleting grammar");
- MemoryBasedBatchGrammar latticeGrammar = new MemoryBasedBatchGrammar("lattice", joshuaConfiguration);
- latticeGrammar.setSpanLimit(-1);
++ MemoryBasedBatchGrammar latticeGrammar = new MemoryBasedBatchGrammar("lattice", joshuaConfiguration, -1);
+ HieroFormatReader reader = new HieroFormatReader();
+
+ String goalNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.goal_symbol);
+ String defaultNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.default_non_terminal);
+
+ //FIXME: too many arguments
+ String ruleString = String.format("[%s] ||| [%s,1] <eps> ||| [%s,1] ||| ", goalNT, goalNT, defaultNT,
+ goalNT, defaultNT);
+
+ Rule rule = reader.parseLine(ruleString);
+ latticeGrammar.addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+
+ this.grammars.add(latticeGrammar);
+ }
+
+ /* Now create a feature function for each owner */
- HashSet<String> ownersSeen = new HashSet<String>();
++ final Set<OwnerId> ownersSeen = new HashSet<OwnerId>();
+
+ for (Grammar grammar: this.grammars) {
- String owner = Vocabulary.word(grammar.getOwner());
++ OwnerId owner = grammar.getOwner();
+ if (! ownersSeen.contains(owner)) {
- this.featureFunctions.add(new PhraseModel(weights, new String[] { "tm", "-owner", owner },
- joshuaConfiguration, grammar));
++ this.featureFunctions.add(
++ new PhraseModel(
++ weights, new String[] { "tm", "-owner", getOwner(owner) }, joshuaConfiguration, grammar));
+ ownersSeen.add(owner);
+ }
+ }
+
+ LOG.info("Memory used {} MB",
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0));
+ }
+
+ /**
+ * Checks if multiple packedGrammars have the same vocabulary by comparing their vocabulary file checksums.
+ */
+ private static void checkSharedVocabularyChecksumsForPackedGrammars(final List<PackedGrammar> packed_grammars) {
+ String previous_checksum = "";
+ for (PackedGrammar grammar : packed_grammars) {
+ final String checksum = grammar.computeVocabularyChecksum();
+ if (previous_checksum.isEmpty()) {
+ previous_checksum = checksum;
+ } else {
+ if (!checksum.equals(previous_checksum)) {
+ throw new RuntimeException(
+ "Trying to load multiple packed grammars with different vocabularies!" +
+ "Have you packed them jointly?");
+ }
+ previous_checksum = checksum;
+ }
+ }
+ }
+
+ /*
+ * This function reads the weights for the model. Feature names and their weights are listed one
+ * per line in the following format:
+ *
+ * FEATURE_NAME WEIGHT
+ */
+ private void readWeights(String fileName) {
+ Decoder.weights = new FeatureVector();
+
+ if (fileName.equals(""))
+ return;
+
+ try {
+ LineReader lineReader = new LineReader(fileName);
+
+ for (String line : lineReader) {
+ line = line.replaceAll("\\s+", " ");
+
+ if (line.equals("") || line.startsWith("#") || line.startsWith("//")
+ || line.indexOf(' ') == -1)
+ continue;
+
+ String tokens[] = line.split("\\s+");
+ String feature = tokens[0];
+ Float value = Float.parseFloat(tokens[1]);
+
+ // Kludge for compatibility with Moses tuners
+ if (joshuaConfiguration.moses) {
+ feature = demoses(feature);
+ }
+
+ weights.increment(feature, value);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ LOG.info("Read {} weights from file '{}'", weights.size(), fileName);
+ }
+
+ private String demoses(String feature) {
+ if (feature.endsWith("="))
+ feature = feature.replace("=", "");
+ if (feature.equals("OOV_Penalty"))
+ feature = "OOVPenalty";
+ else if (feature.startsWith("tm-") || feature.startsWith("lm-"))
+ feature = feature.replace("-", "_");
+ return feature;
+ }
+
+ /**
+ * Feature functions are instantiated with a line of the form
+ *
+ * <pre>
+ * FEATURE OPTIONS
+ * </pre>
+ *
+ * Weights for features are listed separately.
+ *
+ * @throws IOException
+ *
+ */
+ private void initializeFeatureFunctions() throws IOException {
+
+ for (String featureLine : joshuaConfiguration.features) {
+ // line starts with NAME, followed by args
+ // 1. create new class named NAME, pass it config, weights, and the args
+
+ String fields[] = featureLine.split("\\s+");
+ String featureName = fields[0];
+
+ try {
+
+ Class<?> clas = getFeatureFunctionClass(featureName);
+ Constructor<?> constructor = clas.getConstructor(FeatureVector.class,
+ String[].class, JoshuaConfiguration.class);
+ FeatureFunction feature = (FeatureFunction) constructor.newInstance(weights, fields, joshuaConfiguration);
+ this.featureFunctions.add(feature);
+
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Unable to instantiate feature function '%s'!", featureLine), e);
+ }
+ }
+
+ for (FeatureFunction feature : featureFunctions) {
+ LOG.info("FEATURE: {}", feature.logString());
+ }
+
+ weights.registerDenseFeatures(featureFunctions);
+ }
+
+ /**
+ * Searches a list of predefined paths for classes, and returns the first one found. Meant for
+ * instantiating feature functions.
+ *
+ * @param name
+ * @return the class, found in one of the search paths
+ * @throws ClassNotFoundException
+ */
+ private Class<?> getFeatureFunctionClass(String featureName) {
+ Class<?> clas = null;
+
+ String[] packages = { "org.apache.joshua.decoder.ff", "org.apache.joshua.decoder.ff.lm", "org.apache.joshua.decoder.ff.phrase" };
+ for (String path : packages) {
+ try {
+ clas = Class.forName(String.format("%s.%s", path, featureName));
+ break;
+ } catch (ClassNotFoundException e) {
+ try {
+ clas = Class.forName(String.format("%s.%sFF", path, featureName));
+ break;
+ } catch (ClassNotFoundException e2) {
+ // do nothing
+ }
+ }
+ }
+ return clas;
+ }
+
+ /**
+ * Adds a rule to the custom grammar.
+ *
+ * @param rule the rule to add
+ */
+ public void addCustomRule(Rule rule) {
+ customPhraseTable.addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+ }
+
+ public Grammar getCustomPhraseTable() {
+ return customPhraseTable;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
index 0000000,2faacf2..b44a7f0
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
@@@ -1,0 -1,164 +1,157 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder;
+
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+ import org.apache.joshua.decoder.io.DeNormalize;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.decoder.segment_file.Token;
+ import org.apache.joshua.util.FormatUtils;
+
+ /**
+ * A StructuredTranslation instance provides a more structured access to
+ * translation results than the string-based Translation class.
+ * This is useful if the decoder is encapsulated in a larger project, instead
+ * of simply writing to a file or stdout.
+ * StructuredTranslation encodes all relevant information about a derivation,
+ * namely output string, tokens, score, features, and word alignment.
+ *
+ * @author fhieber
+ */
+ public class StructuredTranslation {
+
+ private final Sentence sourceSentence;
+ private final String translationString;
+ private final List<String> translationTokens;
+ private final float translationScore;
+ private final List<List<Integer>> translationWordAlignments;
- private final FeatureVector translationFeatures;
++ private final Map<String,Float> translationFeatures;
+ private final float extractionTime;
+
+ public StructuredTranslation(
+ final Sentence sourceSentence,
+ final String translationString,
+ final List<String> translationTokens,
+ final float translationScore,
+ final List<List<Integer>> translationWordAlignments,
- final FeatureVector translationFeatures,
++ final Map<String,Float> translationFeatures,
+ final float extractionTime) {
+ this.sourceSentence = sourceSentence;
+ this.translationString = translationString;
+ this.translationTokens = translationTokens;
+ this.translationScore = translationScore;
+ this.translationWordAlignments = translationWordAlignments;
+ this.translationFeatures = translationFeatures;
+ this.extractionTime = extractionTime;
+ }
+
+ public Sentence getSourceSentence() {
+ return sourceSentence;
+ }
+
+ public int getSentenceId() {
+ return sourceSentence.id();
+ }
+
+ /**
+ * Produces the raw translation hypothesis (still tokenized).
+ *
+ * @return the raw translation hypothesis
+ */
+ public String getTranslationString() {
+ return translationString;
+ }
+
+ /**
- * Produces the translation formatted according to the value of {@value JoshuaConfiguration.output_format}.
- * Also includes formatting options such as {@value JoshuaConfiguration.project_case}.
++ * Returns the output string formatted according to JoshuaConfiguration.output_format.
+ *
- * @return
++ * @return the formatted string
+ */
+ public String getFormattedTranslationString() {
- JoshuaConfiguration config = sourceSentence.config;
- String outputString = config.outputFormat
- .replace("%s", getTranslationString())
- .replace("%S", DeNormalize.processSingleLine(maybeProjectCase(getTranslationString())))
- .replace("%i", Integer.toString(getSentenceId()))
- .replace("%f", config.moses ? translationFeatures.mosesString() : translationFeatures.toString())
- .replace("%c", String.format("%.3f", getTranslationScore()));
- return outputString;
++ throw new RuntimeException("Not yet implemented");
+ }
+
+ public List<String> getTranslationTokens() {
+ return translationTokens;
+ }
+
+ public float getTranslationScore() {
+ return translationScore;
+ }
+
+ /**
+ * Returns a list of target to source alignments.
++ *
+ * @return a list of target to source alignments
+ */
+ public List<List<Integer>> getTranslationWordAlignments() {
+ return translationWordAlignments;
+ }
+
+ public Map<String,Float> getTranslationFeatures() {
- return translationFeatures.getMap();
++ return translationFeatures;
+ }
+
+ /**
+ * Time taken to build output information from the hypergraph.
+ * @return the time taken to build output information from the hypergraph
+ */
+ public Float getExtractionTime() {
+ return extractionTime;
+ }
+
+ /**
+ * If requested, projects source-side lettercase to target, and appends the alignment from
+ * to the source-side sentence in ||s.
+ *
+ * @param hypothesis todo
+ * @param state todo
+ * @return source-side lettercase to target, and appends the alignment from to the source-side sentence in ||s
+ */
+ private String maybeProjectCase(String hypothesis) {
+ String output = hypothesis;
+
+ JoshuaConfiguration config = sourceSentence.config;
+ if (config.project_case) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = getTranslationWordAlignments();
+ for (int i = 0; i < points.size(); i++) {
+ List<Integer> target = points.get(i);
+ for (int source: target) {
+ Token token = sourceSentence.getTokens().get(source + 1); // skip <s>
+ String annotation = "";
+ if (token != null && token.getAnnotation("lettercase") != null)
+ annotation = token.getAnnotation("lettercase");
+ if (source != 0 && annotation.equals("upper"))
+ tokens[i] = FormatUtils.capitalize(tokens[i]);
+ else if (annotation.equals("all-upper"))
+ tokens[i] = tokens[i].toUpperCase();
+ }
+ }
+
+ output = String.join(" ", tokens);
+ }
+
+ return output;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
index 0000000,4389135..921771e
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
@@@ -1,0 -1,117 +1,120 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder;
+
+ import static java.util.Arrays.asList;
+ import static java.util.Collections.emptyList;
+ import static java.util.Collections.emptyMap;
+ import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+ import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+ import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignmentList;
+ import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+
+ import java.util.List;
+
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.hypergraph.HyperGraph;
+ import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+
+ /**
+ * This factory provides methods to create StructuredTranslation objects
+ * from either Viterbi derivations or KBest derivations.
+ *
+ * @author fhieber
+ */
+ public class StructuredTranslationFactory {
+
+ /**
- * Returns a StructuredTranslation instance from the Viterbi derivation.
++ * Returns a StructuredTranslation instance from the Viterbi derivation. This is used to
++ * obtain the 1-best hypothesis, since traversing the Viterbi derivation directly on the
++ * hypergraph is considerably faster than instantiating the k-best apparatus.
+ *
+ * @param sourceSentence the source sentence
+ * @param hypergraph the hypergraph object
+ * @param featureFunctions the list of active feature functions
+ * @return A StructuredTranslation object representing the Viterbi derivation.
+ */
+ public static StructuredTranslation fromViterbiDerivation(
+ final Sentence sourceSentence,
+ final HyperGraph hypergraph,
+ final List<FeatureFunction> featureFunctions) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(getViterbiString(hypergraph));
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ extractTranslationScore(hypergraph),
+ getViterbiWordAlignmentList(hypergraph),
- getViterbiFeatures(hypergraph, featureFunctions, sourceSentence),
++ getViterbiFeatures(hypergraph, featureFunctions, sourceSentence).getMap(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation from an empty decoder output
+ * @param sourceSentence the source sentence
+ * @return a StructuredTranslation object
+ */
+ public static StructuredTranslation fromEmptyOutput(final Sentence sourceSentence) {
+ return new StructuredTranslation(
- sourceSentence, "", emptyList(), 0, emptyList(), new FeatureVector(), 0f);
++ sourceSentence, "", emptyList(), 0, emptyList(), emptyMap(), 0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation instance from a KBest DerivationState.
++ *
+ * @param sourceSentence Sentence object representing the source.
+ * @param derivationState the KBest DerivationState.
+ * @return A StructuredTranslation object representing the derivation encoded by derivationState.
+ */
+ public static StructuredTranslation fromKBestDerivation(
+ final Sentence sourceSentence,
+ final DerivationState derivationState) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(derivationState.getHypothesis());
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ derivationState.getModelCost(),
+ derivationState.getWordAlignmentList(),
- derivationState.getFeatures(),
++ derivationState.getFeatures().getMap(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ private static float extractTranslationScore(final HyperGraph hypergraph) {
+ if (hypergraph == null) {
+ return 0;
+ } else {
+ return hypergraph.goalNode.getScore();
+ }
+ }
+
+ private static List<String> extractTranslationTokens(final String translationString) {
+ if (translationString.isEmpty()) {
+ return emptyList();
+ } else {
+ return asList(translationString.split("\\s+"));
+ }
+ }
+
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
index 0000000,d0cd96b..355a6f1
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
@@@ -1,0 -1,746 +1,746 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.chart_parser;
+
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.PriorityQueue;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.DotChart.DotNode;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.SourceDependentFF;
+ import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.RuleCollection;
+ import org.apache.joshua.decoder.ff.tm.Trie;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.hypergraph.HyperGraph;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.decoder.segment_file.Token;
+ import org.apache.joshua.lattice.Arc;
+ import org.apache.joshua.lattice.Lattice;
+ import org.apache.joshua.lattice.Node;
+ import org.apache.joshua.util.ChartSpan;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * Chart class this class implements chart-parsing: (1) seeding the chart (2)
+ * cky main loop over bins, (3) identify applicable rules in each bin
+ *
+ * Note: the combination operation will be done in Cell
+ *
+ * Signatures of class: Cell: i, j SuperNode (used for CKY check): i,j, lhs
+ * HGNode ("or" node): i,j, lhs, edge ngrams HyperEdge ("and" node)
+ *
+ * index of sentences: start from zero index of cell: cell (i,j) represent span
+ * of words indexed [i,j-1] where i is in [0,n-1] and j is in [1,n]
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+ public class Chart {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Chart.class);
+ private final JoshuaConfiguration config;
+ // ===========================================================
+ // Statistics
+ // ===========================================================
+
+ /**
+ * how many items have been pruned away because its cost is greater than the
+ * cutoff in calling chart.add_deduction_in_chart()
+ */
+ int nMerged = 0;
+ int nAdded = 0;
+ int nDotitemAdded = 0; // note: there is no pruning in dot-item
+
+ public Sentence getSentence() {
+ return this.sentence;
+ }
+
+ // ===============================================================
+ // Private instance fields (maybe could be protected instead)
+ // ===============================================================
+ private ChartSpan<Cell> cells; // note that in some cell, it might be null
+ private int sourceLength;
+ private List<FeatureFunction> featureFunctions;
+ private Grammar[] grammars;
+ private DotChart[] dotcharts; // each grammar should have a dotchart associated with it
+ private Cell goalBin;
+ private int goalSymbolID = -1;
+ private Lattice<Token> inputLattice;
+
+ private Sentence sentence = null;
+ // private SyntaxTree parseTree;
+ private StateConstraint stateConstraint;
+
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ /*
+ * TODO: Once the Segment interface is adjusted to provide a Lattice<String>
+ * for the sentence() method, we should just accept a Segment instead of the
+ * sentence, segmentID, and constraintSpans parameters. We have the symbol
+ * table already, so we can do the integerization here instead of in
+ * DecoderThread. GrammarFactory.getGrammarForSentence will want the
+ * integerized sentence as well, but then we'll need to adjust that interface
+ * to deal with (non-trivial) lattices too. Of course, we get passed the
+ * grammars too so we could move all of that into here.
+ */
+
+ public Chart(Sentence sentence, List<FeatureFunction> featureFunctions, Grammar[] grammars,
+ String goalSymbol, JoshuaConfiguration config) {
+ this.config = config;
+ this.inputLattice = sentence.getLattice();
+ this.sourceLength = inputLattice.size() - 1;
+ this.featureFunctions = featureFunctions;
+
+ this.sentence = sentence;
+
+ // TODO: OOV handling no longer handles parse tree input (removed after
+ // commit 748eb69714b26dd67cba8e7c25a294347603bede)
+ // this.parseTree = null;
+ // if (sentence instanceof ParsedSentence)
+ // this.parseTree = ((ParsedSentence) sentence).syntaxTree();
+ //
+ this.cells = new ChartSpan<Cell>(sourceLength, null);
+
+ this.goalSymbolID = Vocabulary.id(goalSymbol);
+ this.goalBin = new Cell(this, this.goalSymbolID);
+
+ /* Create the grammars, leaving space for the OOV grammar. */
+ this.grammars = new Grammar[grammars.length + 1];
+ for (int i = 0; i < grammars.length; i++)
+ this.grammars[i + 1] = grammars[i];
+
- MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", this.config);
++ MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", this.config, 20);
+ AbstractGrammar.addOOVRules(oovGrammar, sentence.getLattice(), featureFunctions,
+ this.config.true_oovs_only);
+ this.grammars[0] = oovGrammar;
+
+ // each grammar will have a dot chart
+ this.dotcharts = new DotChart[this.grammars.length];
+ for (int i = 0; i < this.grammars.length; i++)
+ this.dotcharts[i] = new DotChart(this.inputLattice, this.grammars[i], this);
+
+ // Begin to do initialization work
+
+ stateConstraint = null;
+ if (sentence.target() != null)
+ // stateConstraint = new StateConstraint(sentence.target());
+ stateConstraint = new StateConstraint(Vocabulary.START_SYM + " " + sentence.target() + " "
+ + Vocabulary.STOP_SYM);
+
+ /* Find the SourceDependent feature and give it access to the sentence. */
+ for (FeatureFunction ff : this.featureFunctions)
+ if (ff instanceof SourceDependentFF)
+ ((SourceDependentFF) ff).setSource(sentence);
+
+ LOG.debug("Finished seeding chart.");
+ }
+
+ /**
+ * Manually set the goal symbol ID. The constructor expects a String
+ * representing the goal symbol, but there may be time (say, for example, in
+ * the second pass of a synchronous parse) where we want to set the goal
+ * symbol to a particular ID (regardless of String representation).
+ * <p>
+ * This method should be called before expanding the chart, as chart expansion
+ * depends on the goal symbol ID.
+ *
+ * @param i the id of the goal symbol to use
+ */
+ public void setGoalSymbolID(int i) {
+ this.goalSymbolID = i;
+ this.goalBin = new Cell(this, i);
+ return;
+ }
+
+ // ===============================================================
+ // The primary method for filling in the chart
+ // ===============================================================
+
+ /**
+ * Construct the hypergraph with the help from DotChart using cube pruning.
+ * Cube pruning occurs at the span level, with all completed rules from the
+ * dot chart competing against each other; that is, rules with different
+ * source sides *and* rules sharing a source side but with different target
+ * sides are all in competition with each other.
+ *
+ * Terminal rules are added to the chart directly.
+ *
+ * Rules with nonterminals are added to the list of candidates. The candidates
+ * list is seeded with the list of all rules and, for each nonterminal in the
+ * rule, the 1-best tail node for that nonterminal and subspan. If the maximum
+ * arity of a rule is R, then the dimension of the hypercube is R + 1, since
+ * the first dimension is used to record the rule.
+ */
+ private void completeSpan(int i, int j) {
+
+ /* STEP 1: create the heap, and seed it with all of the candidate states */
+ PriorityQueue<CubePruneState> candidates = new PriorityQueue<CubePruneState>();
+
+ /*
+ * Look at all the grammars, seeding the chart with completed rules from the
+ * DotChart
+ */
+ for (int g = 0; g < grammars.length; g++) {
+ if (!grammars[g].hasRuleForSpan(i, j, inputLattice.distance(i, j))
+ || null == dotcharts[g].getDotCell(i, j))
+ continue;
+
+ // for each rule with applicable rules
+ for (DotNode dotNode : dotcharts[g].getDotCell(i, j).getDotNodes()) {
+ RuleCollection ruleCollection = dotNode.getRuleCollection();
+ if (ruleCollection == null)
+ continue;
+
+ List<Rule> rules = ruleCollection.getSortedRules(this.featureFunctions);
+ SourcePath sourcePath = dotNode.getSourcePath();
+
+ if (null == rules || rules.size() == 0)
+ continue;
+
+ if (ruleCollection.getArity() == 0) {
+ /*
+ * The total number of arity-0 items (pre-terminal rules) that we add
+ * is controlled by num_translation_options in the configuration.
+ *
+ * We limit the translation options per DotNode; that is, per LHS.
+ */
+ int numTranslationsAdded = 0;
+
+ /* Terminal productions are added directly to the chart */
+ for (Rule rule : rules) {
+
+ if (config.num_translation_options > 0
+ && numTranslationsAdded >= config.num_translation_options) {
+ break;
+ }
+
+ ComputeNodeResult result = new ComputeNodeResult(this.featureFunctions, rule, null, i,
+ j, sourcePath, this.sentence);
+
+ if (stateConstraint == null || stateConstraint.isLegal(result.getDPStates())) {
+ getCell(i, j).addHyperEdgeInCell(result, rule, i, j, null, sourcePath, true);
+ numTranslationsAdded++;
+ }
+ }
+ } else {
+ /* Productions with rank > 0 are subject to cube pruning */
+
+ Rule bestRule = rules.get(0);
+
+ List<HGNode> currentTailNodes = new ArrayList<HGNode>();
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+ for (SuperNode si : superNodes) {
+ currentTailNodes.add(si.nodes.get(0));
+ }
+
+ /*
+ * `ranks` records the current position in the cube. the 0th index is
+ * the rule, and the remaining indices 1..N correspond to the tail
+ * nodes (= nonterminals in the rule). These tail nodes are
+ * represented by SuperNodes, which group together items with the same
+ * nonterminal but different DP state (e.g., language model state)
+ */
+ int[] ranks = new int[1 + superNodes.size()];
+ Arrays.fill(ranks, 1);
+
+ ComputeNodeResult result = new ComputeNodeResult(featureFunctions, bestRule,
+ currentTailNodes, i, j, sourcePath, sentence);
+ CubePruneState bestState = new CubePruneState(result, ranks, rules, currentTailNodes,
+ dotNode);
+ candidates.add(bestState);
+ }
+ }
+ }
+
+ applyCubePruning(i, j, candidates);
+ }
+
+ /**
+ * Applies cube pruning over a span.
+ *
+ * @param i
+ * @param j
+ * @param stateConstraint
+ * @param candidates
+ */
+ private void applyCubePruning(int i, int j, PriorityQueue<CubePruneState> candidates) {
+
+ // System.err.println(String.format("CUBEPRUNE: %d-%d with %d candidates",
+ // i, j, candidates.size()));
+ // for (CubePruneState cand: candidates) {
+ // System.err.println(String.format(" CAND " + cand));
+ // }
+
+ /*
+ * There are multiple ways to reach each point in the cube, so short-circuit
+ * that.
+ */
+ HashSet<CubePruneState> visitedStates = new HashSet<CubePruneState>();
+
+ int popLimit = config.pop_limit;
+ int popCount = 0;
+ while (candidates.size() > 0 && ((++popCount <= popLimit) || popLimit == 0)) {
+ CubePruneState state = candidates.poll();
+
+ DotNode dotNode = state.getDotNode();
+ List<Rule> rules = state.rules;
+ SourcePath sourcePath = dotNode.getSourcePath();
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+
+ /*
+ * Add the hypothesis to the chart. This can only happen if (a) we're not
+ * doing constrained decoding or (b) we are and the state is legal.
+ */
+ if (stateConstraint == null || stateConstraint.isLegal(state.getDPStates())) {
+ getCell(i, j).addHyperEdgeInCell(state.computeNodeResult, state.getRule(), i, j,
+ state.antNodes, sourcePath, true);
+ }
+
+ /*
+ * Expand the hypothesis by walking down a step along each dimension of
+ * the cube, in turn. k = 0 means we extend the rule being used; k > 0
+ * expands the corresponding tail node.
+ */
+
+ for (int k = 0; k < state.ranks.length; k++) {
+
+ /* Copy the current ranks, then extend the one we're looking at. */
+ int[] nextRanks = new int[state.ranks.length];
+ System.arraycopy(state.ranks, 0, nextRanks, 0, state.ranks.length);
+ nextRanks[k]++;
+
+ /*
+ * We might have reached the end of something (list of rules or tail
+ * nodes)
+ */
+ if (k == 0
+ && (nextRanks[k] > rules.size() || (config.num_translation_options > 0 && nextRanks[k] > config.num_translation_options)))
+ continue;
+ else if ((k != 0 && nextRanks[k] > superNodes.get(k - 1).nodes.size()))
+ continue;
+
+ /* Use the updated ranks to assign the next rule and tail node. */
+ Rule nextRule = rules.get(nextRanks[0] - 1);
+ // HGNode[] nextAntNodes = new HGNode[state.antNodes.size()];
+ List<HGNode> nextAntNodes = new ArrayList<HGNode>(state.antNodes.size());
+ for (int x = 0; x < state.ranks.length - 1; x++)
+ nextAntNodes.add(superNodes.get(x).nodes.get(nextRanks[x + 1] - 1));
+
+ /* Create the next state. */
+ CubePruneState nextState = new CubePruneState(new ComputeNodeResult(featureFunctions,
+ nextRule, nextAntNodes, i, j, sourcePath, this.sentence), nextRanks, rules,
+ nextAntNodes, dotNode);
+
+ /* Skip states that have been explored before. */
+ if (visitedStates.contains(nextState))
+ continue;
+
+ visitedStates.add(nextState);
+ candidates.add(nextState);
+ }
+ }
+ }
+
+ /* Create a priority queue of candidates for each span under consideration */
+ private PriorityQueue<CubePruneState>[] allCandidates;
+
+ private ArrayList<SuperNode> nodeStack;
+
+ /**
+ * Translates the sentence using the CKY+ variation proposed in
+ * "A CYK+ Variant for SCFG Decoding Without A Dot Chart" (Sennrich, SSST
+ * 2014).
+ */
+ private int i = -1;
+
+ public HyperGraph expandSansDotChart() {
+ for (i = sourceLength - 1; i >= 0; i--) {
+ allCandidates = new PriorityQueue[sourceLength - i + 2];
+ for (int id = 0; id < allCandidates.length; id++)
+ allCandidates[id] = new PriorityQueue<CubePruneState>();
+
+ nodeStack = new ArrayList<SuperNode>();
+
+ for (int j = i + 1; j <= sourceLength; j++) {
+ if (!sentence.hasPath(i, j))
+ continue;
+
+ for (int g = 0; g < this.grammars.length; g++) {
+ // System.err.println(String.format("\n*** I=%d J=%d GRAMMAR=%d", i, j, g));
+
+ if (j == i + 1) {
+ /* Handle terminals */
+ Node<Token> node = sentence.getNode(i);
+ for (Arc<Token> arc : node.getOutgoingArcs()) {
+ int word = arc.getLabel().getWord();
+ // disallow lattice decoding for now
+ assert arc.getHead().id() == j;
+ Trie trie = this.grammars[g].getTrieRoot().match(word);
+ if (trie != null && trie.hasRules())
+ addToChart(trie, j, false);
+ }
+ } else {
+ /* Recurse for non-terminal case */
+ consume(this.grammars[g].getTrieRoot(), i, j - 1);
+ }
+ }
+
+ // Now that we've accumulated all the candidates, apply cube pruning
+ applyCubePruning(i, j, allCandidates[j - i]);
+
+ // Add unary nodes
+ addUnaryNodes(this.grammars, i, j);
+ }
+ }
+
+ // transition_final: setup a goal item, which may have many deductions
+ if (null == this.cells.get(0, sourceLength)
+ || !this.goalBin.transitToGoal(this.cells.get(0, sourceLength), this.featureFunctions,
+ this.sourceLength)) {
+ LOG.warn("Input {}: Parse failure (either no derivations exist or pruning is too aggressive",
+ sentence.id());
+ return null;
+ }
+
+ return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.sentence);
+ }
+
+ /**
+ * Recursively consumes the trie, following input nodes, finding applicable
+ * rules and adding them to bins for each span for later cube pruning.
+ *
+ * @param dotNode data structure containing information about what's been
+ * already matched
+ * @param l extension point we're looking at
+ *
+ */
+ private void consume(Trie trie, int j, int l) {
+ /*
+ * 1. If the trie node has any rules, we can add them to the collection
+ *
+ * 2. Next, look at all the outgoing nonterminal arcs of the trie node. If
+ * any of them match an existing chart item, then we know we can extend
+ * (i,j) to (i,l). We then recurse for all m from l+1 to n (the end of the
+ * sentence)
+ *
+ * 3. We also try to match terminals if (j + 1 == l)
+ */
+
+ // System.err.println(String.format("CONSUME %s / %d %d %d", dotNode,
+ // dotNode.begin(), dotNode.end(), l));
+
+ // Try to match terminals
+ if (inputLattice.distance(j, l) == 1) {
+ // Get the current sentence node, and explore all outgoing arcs, since we
+ // might be decoding
+ // a lattice. For sentence decoding, this is trivial: there is only one
+ // outgoing arc.
+ Node<Token> inputNode = sentence.getNode(j);
+ for (Arc<Token> arc : inputNode.getOutgoingArcs()) {
+ int word = arc.getLabel().getWord();
+ Trie nextTrie;
+ if ((nextTrie = trie.match(word)) != null) {
+ // add to chart item over (i, l)
+ addToChart(nextTrie, arc.getHead().id(), i == j);
+ }
+ }
+ }
+
+ // Now try to match nonterminals
+ Cell cell = cells.get(j, l);
+ if (cell != null) {
+ for (int id : cell.getKeySet()) { // for each supernode (lhs), see if you
+ // can match a trie
+ Trie nextTrie = trie.match(id);
+ if (nextTrie != null) {
+ SuperNode superNode = cell.getSuperNode(id);
+ nodeStack.add(superNode);
+ addToChart(nextTrie, superNode.end(), i == j);
+ nodeStack.remove(nodeStack.size() - 1);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds all rules at a trie node to the chart, unless its a unary rule. A
+ * unary rule is the first outgoing arc of a grammar's root trie. For
+ * terminals, these are added during the seeding stage; for nonterminals,
+ * these confuse cube pruning and can result in infinite loops, and are
+ * handled separately (see addUnaryNodes());
+ *
+ * @param trie the grammar node
+ * @param isUnary whether the rules at this dotnode are unary
+ */
+ private void addToChart(Trie trie, int j, boolean isUnary) {
+
+ // System.err.println(String.format("ADD TO CHART %s unary=%s", dotNode,
+ // isUnary));
+
+ if (!isUnary && trie.hasRules()) {
+ DotNode dotNode = new DotNode(i, j, trie, new ArrayList<SuperNode>(nodeStack), null);
+
+ addToCandidates(dotNode);
+ }
+
+ for (int l = j + 1; l <= sentence.length(); l++)
+ consume(trie, j, l);
+ }
+
+ /**
+ * Record the completed rule with backpointers for later cube-pruning.
+ *
+ * @param width
+ * @param rules
+ * @param tailNodes
+ */
+ private void addToCandidates(DotNode dotNode) {
+ // System.err.println(String.format("ADD TO CANDIDATES %s AT INDEX %d",
+ // dotNode, dotNode.end() - dotNode.begin()));
+
+ // TODO: one entry per rule, or per rule instantiation (rule together with
+ // unique matching of input)?
+ List<Rule> rules = dotNode.getRuleCollection().getSortedRules(featureFunctions);
+ Rule bestRule = rules.get(0);
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+
+ List<HGNode> tailNodes = new ArrayList<HGNode>();
+ for (SuperNode superNode : superNodes)
+ tailNodes.add(superNode.nodes.get(0));
+
+ int[] ranks = new int[1 + superNodes.size()];
+ Arrays.fill(ranks, 1);
+
+ ComputeNodeResult result = new ComputeNodeResult(featureFunctions, bestRule, tailNodes,
+ dotNode.begin(), dotNode.end(), dotNode.getSourcePath(), sentence);
+ CubePruneState seedState = new CubePruneState(result, ranks, rules, tailNodes, dotNode);
+
+ allCandidates[dotNode.end() - dotNode.begin()].add(seedState);
+ }
+
+ /**
+ * This function performs the main work of decoding.
+ *
+ * @return the hypergraph containing the translated sentence.
+ */
+ public HyperGraph expand() {
+
+ for (int width = 1; width <= sourceLength; width++) {
+ for (int i = 0; i <= sourceLength - width; i++) {
+ int j = i + width;
+ if (LOG.isDebugEnabled())
+ LOG.debug("Processing span ({}, {})", i, j);
+
+ /* Skips spans for which no path exists (possible in lattices). */
+ if (inputLattice.distance(i, j) == Float.POSITIVE_INFINITY) {
+ continue;
+ }
+
+ /*
+ * 1. Expand the dot through all rules. This is a matter of (a) look for
+ * rules over (i,j-1) that need the terminal at (j-1,j) and looking at
+ * all split points k to expand nonterminals.
+ */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Expanding cell");
+ for (int k = 0; k < this.grammars.length; k++) {
+ /**
+ * Each dotChart can act individually (without consulting other
+ * dotCharts) because it either consumes the source input or the
+ * complete nonTerminals, which are both grammar-independent.
+ **/
+ this.dotcharts[k].expandDotCell(i, j);
+ }
+
+ /*
+ * 2. The regular CKY part: add completed items onto the chart via cube
+ * pruning.
+ */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding complete items into chart");
+ completeSpan(i, j);
+
+ /* 3. Process unary rules. */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding unary items into chart");
+ addUnaryNodes(this.grammars, i, j);
+
+ // (4)=== in dot_cell(i,j), add dot-nodes that start from the /complete/
+ // superIterms in
+ // chart_cell(i,j)
+ if (LOG.isDebugEnabled())
+ LOG.debug("Initializing new dot-items that start from complete items in this cell");
+ for (int k = 0; k < this.grammars.length; k++) {
+ if (this.grammars[k].hasRuleForSpan(i, j, inputLattice.distance(i, j))) {
+ this.dotcharts[k].startDotItems(i, j);
+ }
+ }
+
+ /*
+ * 5. Sort the nodes in the cell.
+ *
+ * Sort the nodes in this span, to make them usable for future
+ * applications of cube pruning.
+ */
+ if (null != this.cells.get(i, j)) {
+ this.cells.get(i, j).getSortedNodes();
+ }
+ }
+ }
+
+ logStatistics();
+
+ // transition_final: setup a goal item, which may have many deductions
+ if (null == this.cells.get(0, sourceLength)
+ || !this.goalBin.transitToGoal(this.cells.get(0, sourceLength), this.featureFunctions,
+ this.sourceLength)) {
+ LOG.warn("Input {}: Parse failure (either no derivations exist or pruning is too aggressive",
+ sentence.id());
+ return null;
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Finished expand");
+ return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.sentence);
+ }
+
+ /**
+ * Get the requested cell, creating the entry if it doesn't already exist.
+ *
+ * @param i span start
+ * @param j span end
+ * @return the cell item
+ */
+ public Cell getCell(int i, int j) {
+ assert i >= 0;
+ assert i <= sentence.length();
+ assert i <= j;
+ if (cells.get(i, j) == null)
+ cells.set(i, j, new Cell(this, goalSymbolID));
+
+ return cells.get(i, j);
+ }
+
+ // ===============================================================
+ // Private methods
+ // ===============================================================
+
+ private void logStatistics() {
+ if (LOG.isDebugEnabled())
+ LOG.debug("Input {}: Chart: added {} merged {} dot-items added: {}",
+ this.sentence.id(), this.nAdded, this.nMerged, this.nDotitemAdded);
+ }
+
+ /**
+ * Handles expansion of unary rules. Rules are expanded in an agenda-based
+ * manner to avoid constructing infinite unary chains. Assumes a triangle
+ * inequality of unary rule expansion (e.g., A -> B will always be cheaper
+ * than A -> C -> B), which is not a true assumption.
+ *
+ * @param grammars A list of the grammars for the sentence
+ * @param i
+ * @param j
+ * @return the number of nodes added
+ */
+ private int addUnaryNodes(Grammar[] grammars, int i, int j) {
+
+ Cell chartBin = this.cells.get(i, j);
+ if (null == chartBin) {
+ return 0;
+ }
+ int qtyAdditionsToQueue = 0;
+ ArrayList<HGNode> queue = new ArrayList<HGNode>(chartBin.getSortedNodes());
+ HashSet<Integer> seen_lhs = new HashSet<Integer>();
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding unary to [{}, {}]", i, j);
+
+ while (queue.size() > 0) {
+ HGNode node = queue.remove(0);
+ seen_lhs.add(node.lhs);
+
+ for (Grammar gr : grammars) {
+ if (!gr.hasRuleForSpan(i, j, inputLattice.distance(i, j)))
+ continue;
+
+ /*
+ * Match against the node's LHS, and then make sure the rule collection
+ * has unary rules
+ */
+ Trie childNode = gr.getTrieRoot().match(node.lhs);
+ if (childNode != null && childNode.getRuleCollection() != null
+ && childNode.getRuleCollection().getArity() == 1) {
+
+ ArrayList<HGNode> antecedents = new ArrayList<HGNode>();
+ antecedents.add(node);
+
+ List<Rule> rules = childNode.getRuleCollection().getSortedRules(this.featureFunctions);
+ for (Rule rule : rules) { // for each unary rules
+
+ ComputeNodeResult states = new ComputeNodeResult(this.featureFunctions, rule,
+ antecedents, i, j, new SourcePath(), this.sentence);
+ HGNode resNode = chartBin.addHyperEdgeInCell(states, rule, i, j, antecedents,
+ new SourcePath(), true);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("{}", rule);
+ if (null != resNode && !seen_lhs.contains(resNode.lhs)) {
+ queue.add(resNode);
+ qtyAdditionsToQueue++;
+ }
+ }
+ }
+ }
+ }
+ return qtyAdditionsToQueue;
+ }
+
+ /***
+ * Add a terminal production (X -> english phrase) to the hypergraph.
+ *
+ * @param i the start index
+ * @param j stop index
+ * @param rule the terminal rule applied
+ * @param srcPath the source path cost
+ */
+ public void addAxiom(int i, int j, Rule rule, SourcePath srcPath) {
+ if (null == this.cells.get(i, j)) {
+ this.cells.set(i, j, new Cell(this, this.goalSymbolID));
+ }
+
+ this.cells.get(i, j).addHyperEdgeInCell(
+ new ComputeNodeResult(this.featureFunctions, rule, null, i, j, srcPath, sentence), rule, i,
+ j, null, srcPath, false);
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
index 0000000,ae273b7..f544f50
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
@@@ -1,0 -1,72 +1,73 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import java.util.List;
+
+ import org.apache.joshua.decoder.JoshuaConfiguration;
++import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
-import org.apache.joshua.decoder.chart_parser.SourcePath;
-import org.apache.joshua.corpus.Vocabulary;
+
+ /**
+ * This feature function counts rules from a particular grammar (identified by the owner) having an
+ * arity within a specific range. It expects three parameters upon initialization: the owner, the
+ * minimum arity, and the maximum arity.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li zhifei.work@gmail.com
+ */
+ public class ArityPhrasePenalty extends StatelessFF {
+
+ // when the rule.arity is in the range, then this feature is activated
- private final int owner;
++ private final OwnerId owner;
+ private final int minArity;
+ private final int maxArity;
+
+ public ArityPhrasePenalty(final FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "ArityPenalty", args, config);
+
- this.owner = Vocabulary.id(parsedArgs.get("owner"));
++ this.owner = OwnerMap.register(parsedArgs.get("owner"));
+ this.minArity = Integer.parseInt(parsedArgs.get("min-arity"));
+ this.maxArity = Integer.parseInt(parsedArgs.get("max-arity"));
+ }
+
+ /**
+ * Returns 1 if the arity penalty feature applies to the current rule.
+ */
+ private int isEligible(final Rule rule) {
- if (this.owner == rule.getOwner() && rule.getArity() >= this.minArity
++ if (this.owner.equals(rule.getOwner()) && rule.getArity() >= this.minArity
+ && rule.getArity() <= this.maxArity)
+ return 1;
+
+ return 0;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ acc.add(name, isEligible(rule));
+
+ return null;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
index 0000000,58de5f4..75158d0
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
@@@ -1,0 -1,150 +1,152 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import static com.google.common.cache.CacheBuilder.newBuilder;
+
+ import java.util.ArrayList;
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.util.FormatUtils;
+
+ import com.google.common.cache.Cache;
+
+ /**
+ * Lexical alignment features denoting alignments, deletions, and insertions.
+ */
+ public class LexicalFeatures extends StatelessFF {
+
+ private final boolean useAlignments;
+ private final boolean useDeletions;
+ private final boolean useInsertions;
+
+ private static final String NAME = "LexicalFeatures";
+ // value to fire for features
+ private static final int VALUE = 1;
+ //whether this feature is restricted to a certain grammar/owner
+ private final boolean ownerRestriction;
+ // the grammar/owner this feature is restricted to fire
- private final int owner;
++ private final OwnerId owner;
+ // Strings separating words
+ private static final String SEPARATOR = "~";
+
+ private final Cache<Rule, List<String>> featureCache;
+
+ public LexicalFeatures(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, NAME, args, config);
+
+ ownerRestriction = (parsedArgs.containsKey("owner")) ? true : false;
- owner = ownerRestriction ? Vocabulary.id(parsedArgs.get("owner")) : 0;
++ owner = ownerRestriction ? OwnerMap.register(parsedArgs.get("owner")) : OwnerMap.UNKNOWN_OWNER_ID;
+
+ useAlignments = parsedArgs.containsKey("alignments");
+ useDeletions = parsedArgs.containsKey("deletions");
+ useInsertions = parsedArgs.containsKey("insertions");
+
+ // initialize cache
+ if (parsedArgs.containsKey("cacheSize")) {
+ featureCache = newBuilder().maximumSize(Integer.parseInt(parsedArgs.get("cacheSize"))).build();
+ } else {
+ featureCache = newBuilder().maximumSize(config.cachedRuleSize).build();
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- if (ownerRestriction && rule.getOwner() != owner) {
++ if (ownerRestriction && rule.getOwner().equals(owner)) {
+ return null;
+ }
+
+ List<String> featureNames = featureCache.getIfPresent(rule);
+ if (featureNames == null) {
+ featureNames = getFeatures(rule);
+ featureCache.put(rule, featureNames);
+ }
+ for (String feature : featureNames) {
+ acc.add(feature, VALUE);
+ }
+
+ return null;
+ }
+
+ /**
+ * Obtains the feature ids for the given rule.
+ * @param rule
+ * @return String representing the feature name.s
+ */
+ private List<String> getFeatures(final Rule rule) {
+ final List<String> result = new ArrayList<>();
+
+ byte[] alignments = rule.getAlignment();
+ if (alignments == null) {
+ return result;
+ }
+ int[] sourceWords = rule.getFrench();
+ int[] targetWords = rule.getEnglish();
+
+ // sourceAligned & targetAligned indicate whether an index is covered by alignments
+ boolean[] sourceAligned = new boolean[sourceWords.length];
+ boolean[] targetAligned = new boolean[targetWords.length];
+
+ // translations: aligned words
+ for (int i = 0; i < alignments.length; i+=2) {
+ byte sourceIndex = alignments[i];
+ byte targetIndex = alignments[i + 1];
+ sourceAligned[sourceIndex] = true;
+ targetAligned[targetIndex] = true;
+ if (useAlignments) {
+ result.add(
+ "T:" +
+ Vocabulary.word(sourceWords[sourceIndex]) +
+ SEPARATOR +
+ Vocabulary.word(targetWords[targetIndex]));
+ }
+ }
+
+ // deletions: unaligned source words
+ if (useDeletions) {
+ for (int i = 0; i < sourceAligned.length; i++) {
+ if (!sourceAligned[i] && ! FormatUtils.isNonterminal(sourceWords[i])) {
+ result.add("D:" + Vocabulary.word(sourceWords[i]));
+ }
+ }
+ }
+
+ // insertions: unaligned target words
+ if (useInsertions) {
+ for (int i = 0; i < targetAligned.length; i++) {
+ if (useInsertions && !targetAligned[i] && ! FormatUtils.isNonterminal(targetWords[i])) {
+ result.add("I:" + Vocabulary.word(targetWords[i]));
+ }
+ }
+ }
+
+ return result;
+ }
+ }