You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by le...@apache.org on 2016/05/16 06:26:33 UTC
[17/66] [partial] incubator-joshua git commit: JOSHUA-252 Make it
possible to use Maven to build Joshua
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java b/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java
new file mode 100644
index 0000000..40b92b3
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * This class defines Joshua's feature function interface, for both sparse and
+ * dense features. It is immediately inherited by StatelessFF and StatefulFF,
+ * which provide functionality common to stateless and stateful features,
+ * respectively. Any feature implementation should extend those classes, and not
+ * this one. The distinction between stateless and stateful features is somewhat
+ * narrow: all features have the opportunity to return an instance of a
+ * {@link DPState} object, and stateless ones just return null.
+ *
+ * Features in Joshua work like templates. Each feature function defines any
+ * number of actual features, which are associated with weights. The task of the
+ * feature function is to compute the features that are fired in different
+ * circumstances and then return the inner product of those features with the
+ * weight vector. Feature functions can also produce estimates of their future
+ * cost (via {@link estimateCost()}); these values are not used in computing the
+ * score, but are only used for sorting rules during cube pruning. The
+ * individual features produced by each template should have globally unique
+ * names; a good convention is to prefix each feature with the name of the
+ * template that produced it.
+ *
+ * Joshua does not retain individual feature values while decoding, since this
+ * requires keeping a sparse feature vector along every hyperedge, which can be
+ * expensive. Instead, it computes only the weighted cost of each edge. If the
+ * individual feature values are requested, the feature functions are replayed
+ * in post-processing, say during k-best list extraction. This is implemented in
+ * a generic way by passing an {@link Accumulator} object to the compute()
+ * function. During decoding, the accumulator simply sums weighted features in a
+ * scalar. During k-best extraction, when individual feature values are needed,
+ * a {@link FeatureAccumulator} is used to retain the individual values.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Juri Ganitkevich <ju...@cs.jhu.edu>
+ */
+public abstract class FeatureFunction {
+
+ /*
+ * The name of the feature function; this generally matches the weight name on
+ * the config file. This can also be used as a prefix for feature / weight
+ * names, for templates that define multiple features.
+ */
+ protected String name = null;
+
+ /*
+ * The list of features each function can contribute, along with the dense feature IDs.
+ */
+ protected String[] denseFeatureNames = null;
+ protected int[] denseFeatureIDs = null;
+
+ /*
+ * The first dense feature index
+ */
+ protected int denseFeatureIndex = -1;
+
+ // The list of arguments passed to the feature, and the hash for the parsed args
+ protected String[] args;
+ protected HashMap<String, String> parsedArgs = null;
+
+ /*
+ * The global weight vector used by the decoder, passed it when the feature is
+ * instantiated
+ */
+ protected FeatureVector weights;
+
+ /* The config */
+ protected JoshuaConfiguration config;
+
+ public String getName() {
+ return name;
+ }
+
+ // Whether the feature has state.
+ public abstract boolean isStateful();
+
+ public FeatureFunction(FeatureVector weights, String name, String[] args, JoshuaConfiguration config) {
+ this.weights = weights;
+ this.name = name;
+ this.args = args;
+ this.config = config;
+
+ this.parsedArgs = FeatureFunction.parseArgs(args);
+ }
+
+ /**
+ * Any feature function can use this to report dense features names to the master code. The
+ * parameter tells the feature function the index of the first available dense feature ID; the feature
+ * function will then use IDs (id..id+names.size()-1).
+ *
+ * @param id the id of the first dense feature id to use
+ * @return a list of dense feature names
+ */
+ public ArrayList<String> reportDenseFeatures(int id) {
+ return new ArrayList<String>();
+ }
+
+ public String logString() {
+ try {
+ return String.format("%s (weight %.3f)", name, weights.getSparse(name));
+ } catch (RuntimeException e) {
+ return name;
+ }
+ }
+
+ /**
+ * This is the main function for defining feature values. The implementor
+ * should compute all the features along the hyperedge, calling acc.put(name,
+ * value) for each feature. It then returns the newly-computed dynamic
+ * programming state for this feature (for example, for the
+ * {@link LanguageModelFF} feature, this returns the new language model
+ * context). For stateless features, this value is null.
+ *
+ * Note that the accumulator accumulates *unweighted* feature values. The
+ * feature vector is multiplied times the weight vector later on.
+ *
+ * @param rule
+ * @param tailNodes
+ * @param i
+ * @param j
+ * @param sourcePath
+ * @param sentID
+ * @param acc
+ * @return the new dynamic programming state (null for stateless features)
+ */
+ public abstract DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc);
+
+ /**
+ * Feature functions must overrided this. StatefulFF and StatelessFF provide
+ * reasonable defaults since most features do not fire on the goal node.
+ *
+ * @param tailNode
+ * @param i
+ * @param j
+ * @param sourcePath
+ * @param sentID
+ * @param acc
+ * @return the DPState (null if none)
+ */
+ public abstract DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc);
+
+ /**
+ * This is a convenience function for retrieving the features fired when
+ * applying a rule, provided for backward compatibility.
+ *
+ * Returns the *unweighted* cost of the features delta computed at this
+ * position. Note that this is a feature delta, so existing feature costs of
+ * the tail nodes should not be incorporated, and it is very important not to
+ * incorporate the feature weights. This function is used in the kbest
+ * extraction code but could also be used in computing the cost.
+ *
+ * @param rule
+ * @param tailNodes
+ * @param i
+ * @param j
+ * @param sourcePath
+ * @param sentID
+ * @return an *unweighted* feature delta
+ */
+ public final FeatureVector computeFeatures(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence) {
+
+ FeatureAccumulator features = new FeatureAccumulator();
+ compute(rule, tailNodes, i, j, sourcePath, sentence, features);
+ return features.getFeatures();
+ }
+
+ /**
+ * This function is called for the final transition. For example, the
+ * LanguageModel feature function treats the last rule specially. It needs to
+ * return the *weighted* cost of applying the feature. Provided for backward
+ * compatibility.
+ *
+ * @param tailNode
+ * @param i
+ * @param j
+ * @param sourcePath
+ * @param sentID
+ * @return a *weighted* feature cost
+ */
+ public final float computeFinalCost(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence) {
+
+ ScoreAccumulator score = new ScoreAccumulator();
+ computeFinal(tailNode, i, j, sourcePath, sentence, score);
+ return score.getScore();
+ }
+
+ /**
+ * Returns the *unweighted* feature delta for the final transition (e.g., for
+ * the language model feature function). Provided for backward compatibility.
+ *
+ * @param tailNode
+ * @param i
+ * @param j
+ * @param sourcePath
+ * @param sentID
+ * @return
+ */
+ public final FeatureVector computeFinalFeatures(HGNode tailNode, int i, int j,
+ SourcePath sourcePath, Sentence sentence) {
+
+ FeatureAccumulator features = new FeatureAccumulator();
+ computeFinal(tailNode, i, j, sourcePath, sentence, features);
+ return features.getFeatures();
+ }
+
+ /**
+ * This function is called when sorting rules for cube pruning. It must return
+ * the *weighted* estimated cost of applying a feature. This need not be the
+ * actual cost of applying the rule in context. Basically, it's the inner
+ * product of the weight vector and all features found in the grammar rule,
+ * though some features (like LanguageModelFF) can also compute some of their
+ * values. This is just an estimate of the cost, which helps do better
+ * sorting. Later, the real cost of this feature function is called via
+ * compute();
+ *
+ * @return the *weighted* cost of applying the feature.
+ */
+ public abstract float estimateCost(Rule rule, Sentence sentence);
+
+ /**
+ * This feature is called to produce a *weighted estimate* of the future cost
+ * of applying this feature. This value is not incorporated into the model
+ * score but is used in pruning decisions. Stateless features return 0.0f by
+ * default, but Stateful features might want to override this.
+ *
+ * @param rule
+ * @param state
+ * @param sentence
+ * @return the *weighted* future cost estimate of applying this rule in
+ * context.
+ */
+ public abstract float estimateFutureCost(Rule rule, DPState state, Sentence sentence);
+
+ /**
+ * Parses the arguments passed to a feature function in the Joshua config file TODO: Replace this
+ * with a proper CLI library at some point Expects key value pairs in the form : -argname value
+ * Any key without a value is added with an empty string as value Multiple values for the same key
+ * are not parsed. The first one is used.
+ *
+ * @param rawArgs A string with the raw arguments and their names
+ * @return A hash with the keys and the values of the string
+ */
+ public static HashMap<String, String> parseArgs(String[] args) {
+ HashMap<String, String> parsedArgs = new HashMap<String, String>();
+ boolean lookingForValue = false;
+ String currentKey = "";
+ for (int i = 0; i < args.length; i++) {
+
+ Pattern argKeyPattern = Pattern.compile("^-[a-zA-Z]\\S+");
+ Matcher argKey = argKeyPattern.matcher(args[i]);
+ if (argKey.find()) {
+ // This is a key
+ // First check to see if there is a key that is waiting to be written
+ if (lookingForValue) {
+ // This is a key with no specified value
+ parsedArgs.put(currentKey, "");
+ }
+ // Now store the new key and look for its value
+ currentKey = args[i].substring(1);
+ lookingForValue = true;
+ } else {
+ // This is a value
+ if (lookingForValue) {
+ parsedArgs.put(currentKey, args[i]);
+ lookingForValue = false;
+ }
+ }
+ }
+ return parsedArgs;
+ }
+
+ /**
+ * Accumulator objects allow us to generalize feature computation.
+ * ScoreAccumulator takes (feature,value) pairs and simple stores the weighted
+ * sum (for decoding). FeatureAccumulator records the named feature values
+ * (for k-best extraction).
+ */
+
+ public interface Accumulator {
+ public void add(String name, float value);
+ public void add(int id, float value);
+ }
+
+ public class ScoreAccumulator implements Accumulator {
+ private float score;
+
+ public ScoreAccumulator() {
+ this.score = 0.0f;
+ }
+
+ @Override
+ public void add(String name, float value) {
+ score += value * weights.getSparse(name);
+ }
+
+ @Override
+ public void add(int id, float value) {
+ score += value * weights.getDense(id);
+ }
+
+ public float getScore() {
+ return score;
+ }
+ }
+
+ public class FeatureAccumulator implements Accumulator {
+ private FeatureVector features;
+
+ public FeatureAccumulator() {
+ this.features = new FeatureVector();
+ }
+
+ @Override
+ public void add(String name, float value) {
+ features.increment(name, value);
+ }
+
+ @Override
+ public void add(int id, float value) {
+ features.increment(id, value);
+ }
+
+ public FeatureVector getFeatures() {
+ return features;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java b/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
new file mode 100644
index 0000000..dcbcda2
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An implementation of a sparse feature vector, using for representing both weights and feature
+ * values.
+ *
+ * This class is used to hold both the decoder weights and the feature values accumulated across
+ * each edge. When features are read in upon decoder startup, they all start out as sparse features
+ * and are stored in the hash table. After the feature functions have been loaded, the decoder
+ * queries each of them for their sparse features via {@link registerDenseFeatures}. Those features
+ * returned by each decoder are then *removed* from the sparse feature hash and placed in the dense
+ * feature array. Therefore, when a feature registers a dense feature, it should take care to
+ * query either {@link getDense()} or {@link getSparse} when asking for the feature values later on.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+
+public class FeatureVector {
+ /*
+ * A list of the dense feature names. Increased via calls to registerDenseFeatures()
+ */
+ public static ArrayList<String> DENSE_FEATURE_NAMES = new ArrayList<String>();
+
+ /*
+ * The values of each of the dense features, defaulting to 0.
+ */
+ private ArrayList<Float> denseFeatures = null;
+
+ /*
+ * Value of sparse features.
+ */
+ private HashMap<String, Float> sparseFeatures;
+
+ public FeatureVector() {
+ sparseFeatures = new HashMap<String, Float>();
+ denseFeatures = new ArrayList<Float>(DENSE_FEATURE_NAMES.size());
+ for (int i = 0; i < denseFeatures.size(); i++)
+ denseFeatures.set(i, 0.0f);
+ }
+
+ /**
+ * This version of the constructor takes an uninitialized feature with potentially intermingled
+ * labeled and unlabeled feature values, of the format:
+ *
+ * [feature1=]value [feature2=]value
+ *
+ * It produces a Feature Vector where all unlabeled features have been labeled by appending the
+ * unlabeled feature index (starting at 0) to the defaultPrefix value.
+ *
+ * **IMPORTANT** The feature values are inverted, for historical reasons, which leads to a lot
+ * of confusion. They have to be inverted here and when the score is actually computed. They
+ * are inverted here (which is used to build the feature vector representation of a rule's dense
+ * features) and in {@link BilingualRule::estimateRuleCost()}, where the rule's precomputable
+ * (weighted) score is cached.
+ *
+ * @param featureString, the string of labeled and unlabeled features (probably straight from the
+ * grammar text file)
+ * @param prefix, the prefix to use for unlabeled features (probably "tm_OWNER_")
+ */
+ public FeatureVector(String featureString, String prefix) {
+
+// System.err.println(String.format("FEATURES_OF(%s, %s)", featureString, prefix));
+
+ /*
+ * Read through the features on this rule, adding them to the feature vector. Unlabeled features
+ * are converted to a canonical form.
+ *
+ * Note that it's bad form to mix unlabeled features and the named feature index they are mapped
+ * to, but we are being liberal in what we accept.
+ *
+ * IMPORTANT: Note that, for historical reasons, the sign is reversed on all *dense* scores.
+ * This is the source of *no end* of confusion and should be done away with.
+ */
+ this();
+
+ int denseFeatureIndex = 0;
+
+ if (!featureString.trim().equals("")) {
+ for (String token : featureString.split("\\s+")) {
+ if (token.indexOf('=') == -1) {
+ /*
+ * If we encounter an unlabeled feature, it is the next dense feature
+ */
+ while (denseFeatures.size() <= denseFeatureIndex)
+ denseFeatures.add(0.0f);
+ denseFeatures.set(denseFeatureIndex, -Float.parseFloat(token));
+ denseFeatureIndex++;
+ } else {
+ /*
+ * Labeled features are of two types: if they start with the prefix, they are actually
+ * dense feature in disguise; otherwise, they are proper sparse features.
+ */
+ int splitPoint = token.indexOf('=');
+ if (token.startsWith(prefix)) {
+// System.err.println(String.format(" PREFIX=%s '%s'.substring(%d,%d) = %s", prefix, token, prefix.length(), splitPoint,
+// token.substring(prefix.length(), splitPoint)));
+ int index = Integer.parseInt(token.substring(prefix.length(), splitPoint));
+ while (denseFeatures.size() <= index)
+ denseFeatures.add(0.0f);
+ denseFeatures.set(index, 1.0f * Float.parseFloat(token.substring(splitPoint + 1)));
+ } else {
+ sparseFeatures.put(token.substring(0, splitPoint),
+ Float.parseFloat(token.substring(splitPoint + 1)));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Register one or more dense features with the global weight vector. This assumes them global
+ * IDs, and then returns the index of the first feature (from which the calling feature function
+ * can infer them all). This *must* be called by every feature function wishing to register
+ * dense features!
+ *
+ * @param names
+ * @return
+ */
+ public void registerDenseFeatures(ArrayList<FeatureFunction> featureFunctions) {
+ for (FeatureFunction feature: featureFunctions) {
+ ArrayList<String> names = feature.reportDenseFeatures(denseFeatures.size());
+ for (String name: names) {
+ DENSE_FEATURE_NAMES.add(name);
+ denseFeatures.add(getSparse(name));
+ sparseFeatures.remove(name);
+ }
+ }
+ }
+
+ public ArrayList<Float> getDenseFeatures() {
+ return denseFeatures;
+ }
+
+ public HashMap<String,Float> getSparseFeatures() {
+ return sparseFeatures;
+ }
+
+ public Set<String> keySet() {
+ return sparseFeatures.keySet();
+ }
+
+ public int size() {
+ return sparseFeatures.size() + denseFeatures.size();
+ }
+
+ public FeatureVector clone() {
+ FeatureVector newOne = new FeatureVector();
+ for (String key : this.sparseFeatures.keySet())
+ newOne.set(key, this.sparseFeatures.get(key));
+ for (int i = 0; i < denseFeatures.size(); i++)
+ newOne.set(i, getDense(i));
+ return newOne;
+ }
+
+ /**
+ * Subtracts the weights in the other feature vector from this one. Note that this is not set
+ * subtraction; keys found in the other FeatureVector but not in this one will be initialized with
+ * a value of 0.0f before subtraction.
+ */
+ public void subtract(FeatureVector other) {
+ for (int i = 0; i < denseFeatures.size(); i++)
+ denseFeatures.set(i, getDense(i) - other.getDense(i));
+
+ for (String key : other.keySet()) {
+ float oldValue = (sparseFeatures.containsKey(key)) ? sparseFeatures.get(key) : 0.0f;
+ sparseFeatures.put(key, oldValue - other.getSparse(key));
+ }
+ }
+
+ /**
+ * Adds the weights in the other feature vector to this one. This is set union, with values shared
+ * between the two being summed.
+ */
+ public void add(FeatureVector other) {
+ while (denseFeatures.size() < other.denseFeatures.size())
+ denseFeatures.add(0.0f);
+
+ for (int i = 0; i < other.denseFeatures.size(); i++)
+ increment(i, other.getDense(i));
+
+ for (String key : other.keySet()) {
+ if (!sparseFeatures.containsKey(key))
+ sparseFeatures.put(key, other.getSparse(key));
+ else
+ sparseFeatures.put(key, sparseFeatures.get(key) + other.getSparse(key));
+ }
+ }
+
+ /**
+ * Return the weight of a feature by name, after checking to determine if it is sparse or dense.
+ *
+ */
+ public float getWeight(String feature) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ if (DENSE_FEATURE_NAMES.get(i).equals(feature)) {
+ return getDense(i);
+ }
+ }
+ return getSparse(feature);
+ }
+
+ /**
+ * Return the weight of a sparse feature, indexed by its name.
+ *
+ * @param feature
+ * @return the sparse feature's weight, or 0 if not found.
+ */
+ public float getSparse(String feature) {
+ if (sparseFeatures.containsKey(feature))
+ return sparseFeatures.get(feature);
+ return 0.0f;
+ }
+
+ public boolean hasValue(String name) {
+ return sparseFeatures.containsKey(name);
+ }
+
+ /**
+ * Return the weight of a dense feature, indexed by its feature index, or 0.0f, if the feature
+ * is not found. In other words, this is a safe way to query the dense feature vector.
+ *
+ * @param id
+ * @return the dense feature's value, or 0 if not found.
+ */
+ public float getDense(int id) {
+ if (id < denseFeatures.size())
+ return denseFeatures.get(id);
+ return 0.0f;
+ }
+
+ public void increment(String feature, float value) {
+ sparseFeatures.put(feature, getSparse(feature) + value);
+ }
+
+ public void increment(int id, float value) {
+ while (id >= denseFeatures.size())
+ denseFeatures.add(0.0f);
+ denseFeatures.set(id, getDense(id) + value);
+ }
+
+ /**
+ * Set the value of a feature. We need to first determine whether the feature is a dense or
+ * sparse one, then set accordingly.
+ *
+ * @param feature
+ * @param value
+ */
+ public void set(String feature, float value) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ if (DENSE_FEATURE_NAMES.get(i).equals(feature)) {
+ denseFeatures.set(i, value);
+ return;
+ }
+ }
+ // No dense feature was found; assume it's sparse
+ sparseFeatures.put(feature, value);
+ }
+
+ public void set(int id, float value) {
+ while (id >= denseFeatures.size())
+ denseFeatures.add(0.0f);
+ denseFeatures.set(id, value);
+ }
+
+ public Map<String, Float> getMap() {
+ return sparseFeatures;
+ }
+
+ /**
+ * Computes the inner product between this feature vector and another one.
+ */
+ public float innerProduct(FeatureVector other) {
+ float cost = 0.0f;
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++)
+ cost += getDense(i) * other.getDense(i);
+
+ for (String key : sparseFeatures.keySet())
+ cost += sparseFeatures.get(key) * other.getSparse(key);
+
+ return cost;
+ }
+
+ public void times(float value) {
+ for (String key : sparseFeatures.keySet())
+ sparseFeatures.put(key, sparseFeatures.get(key) * value);
+ }
+
+ /***
+ * Moses distinguishes sparse features as those containing an underscore, so we have to fake it
+ * to be compatible with their tuners.
+ */
+ public String mosesString() {
+ StringBuilder outputString = new StringBuilder();
+
+ HashSet<String> printed_keys = new HashSet<String>();
+
+ // First print all the dense feature names in order
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ outputString.append(String.format("%s=%.3f ", DENSE_FEATURE_NAMES.get(i).replaceAll("_", "-"), getDense(i)));
+ printed_keys.add(DENSE_FEATURE_NAMES.get(i));
+ }
+
+ // Now print the sparse features
+ ArrayList<String> keys = new ArrayList<String>(sparseFeatures.keySet());
+ Collections.sort(keys);
+ for (String key: keys) {
+ if (! printed_keys.contains(key)) {
+ float value = sparseFeatures.get(key);
+ if (key.equals("OOVPenalty"))
+ // force moses to see it as sparse
+ key = "OOV_Penalty";
+ outputString.append(String.format("%s=%.3f ", key, value));
+ }
+ }
+ return outputString.toString().trim();
+ }
+
+ /***
+ * Outputs a list of feature names. All dense features are printed. Feature names are printed
+ * in the order they were read in.
+ */
+ @Override
+ public String toString() {
+ StringBuilder outputString = new StringBuilder();
+
+ HashSet<String> printed_keys = new HashSet<String>();
+
+ // First print all the dense feature names in order
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ outputString.append(String.format("%s=%.3f ", DENSE_FEATURE_NAMES.get(i), getDense(i)));
+ printed_keys.add(DENSE_FEATURE_NAMES.get(i));
+ }
+
+ // Now print the rest of the features
+ ArrayList<String> keys = new ArrayList<String>(sparseFeatures.keySet());
+ Collections.sort(keys);
+ for (String key: keys)
+ if (! printed_keys.contains(key))
+ outputString.append(String.format("%s=%.3f ", key, sparseFeatures.get(key)));
+
+ return outputString.toString().trim();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java b/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java
new file mode 100644
index 0000000..38a85db
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+/***
+ * @author Gideon Wenniger
+ */
+
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+public class LabelCombinationFF extends StatelessFF {
+
+ public LabelCombinationFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "LabelCombination", args, config);
+ }
+
+ public String getLowerCasedFeatureName() {
+ return name.toLowerCase();
+ }
+
+ private final String computeRuleLabelCombinationDescriptor(Rule rule) {
+ StringBuilder result = new StringBuilder(getLowerCasedFeatureName() + "_");
+ result.append(RulePropertiesQuerying.getLHSAsString(rule));
+ // System.out.println("Rule: " + rule);
+ for (String foreignNonterminalString : RulePropertiesQuerying.getRuleSourceNonterminalStrings(rule)) {
+ result.append("_").append(foreignNonterminalString);
+ }
+ return result.toString();
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ if (rule != null)
+ acc.add(computeRuleLabelCombinationDescriptor(rule), 1);
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java b/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java
new file mode 100644
index 0000000..0f70372
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+/***
+ * @author Gideon Wenniger
+ */
+
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+import joshua.util.ListUtil;
+
+public class LabelSubstitutionFF extends StatelessFF {
+ private static final String MATCH_SUFFIX = "MATCH";
+ private static final String NO_MATCH_SUFFIX = "NOMATCH";
+
+ public LabelSubstitutionFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "LabelSubstitution", args, config);
+ }
+
+ public String getLowerCasedFeatureName() {
+ return name.toLowerCase();
+ }
+
+ public String getMatchFeatureSuffix(String ruleNonterminal, String substitutionNonterminal) {
+ if (ruleNonterminal.equals(substitutionNonterminal)) {
+ return MATCH_SUFFIX;
+ } else {
+ return NO_MATCH_SUFFIX;
+ }
+ }
+
+ public static String getSubstitutionSuffix(String ruleNonterminal, String substitutionNonterminal) {
+ return substitutionNonterminal + "_substitutes_" + ruleNonterminal;
+ }
+
+ private final String computeLabelMatchingFeature(String ruleNonterminal,
+ String substitutionNonterminal) {
+ String result = getLowerCasedFeatureName() + "_";
+ result += getMatchFeatureSuffix(ruleNonterminal, substitutionNonterminal);
+ return result;
+ }
+
+ private final String computeLabelSubstitutionFeature(String ruleNonterminal,
+ String substitutionNonterminal) {
+ String result = getLowerCasedFeatureName() + "_";
+ result += getSubstitutionSuffix(ruleNonterminal, substitutionNonterminal);
+ return result;
+ }
+
+ private static final String getRuleLabelsDescriptorString(Rule rule) {
+ String result = "";
+ String leftHandSide = RulePropertiesQuerying.getLHSAsString(rule);
+ List<String> ruleSourceNonterminals = RulePropertiesQuerying
+ .getRuleSourceNonterminalStrings(rule);
+ boolean isInverting = rule.isInverting();
+ result += "<LHS>" + leftHandSide + "</LHS>";
+ result += "_<Nont>";
+ result += ListUtil.stringListStringWithoutBracketsCommaSeparated(ruleSourceNonterminals);
+ result += "</Nont>";
+ if(isInverting)
+ {
+ result += "_INV";
+ }
+ else
+ {
+ result += "_MONO";
+ }
+
+ return result;
+ }
+
+ private static final String getSubstitutionsDescriptorString(List<HGNode> tailNodes) {
+ String result = "_<Subst>";
+ List<String> substitutionNonterminals = RulePropertiesQuerying
+ .getSourceNonterminalStrings(tailNodes);
+ result += ListUtil.stringListStringWithoutBracketsCommaSeparated(substitutionNonterminals);
+ result += "</Subst>";
+ return result;
+ }
+
+ public final String getGapLabelsForRuleSubstitutionSuffix(Rule rule, List<HGNode> tailNodes) {
+ String result = getLowerCasedFeatureName() + "_";
+ result += getRuleLabelsDescriptorString(rule);
+ result += getSubstitutionsDescriptorString(tailNodes);
+ return result;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ if (rule != null && (tailNodes != null)) {
+
+ List<String> ruleSourceNonterminals = RulePropertiesQuerying
+ .getRuleSourceNonterminalStrings(rule);
+ List<String> substitutionNonterminals = RulePropertiesQuerying
+ .getSourceNonterminalStrings(tailNodes);
+ // Assert.assertEquals(ruleSourceNonterminals.size(), substitutionNonterminals.size());
+ for (int nonterinalIndex = 0; nonterinalIndex < ruleSourceNonterminals.size(); nonterinalIndex++) {
+ String ruleNonterminal = ruleSourceNonterminals.get(nonterinalIndex);
+ String substitutionNonterminal = substitutionNonterminals.get(nonterinalIndex);
+ acc.add(computeLabelMatchingFeature(ruleNonterminal, substitutionNonterminal), 1);
+ acc.add(computeLabelSubstitutionFeature(ruleNonterminal, substitutionNonterminal), 1);
+ }
+ acc.add(getGapLabelsForRuleSubstitutionSuffix(rule, tailNodes), 1);
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java b/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
new file mode 100644
index 0000000..6a06548
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.JoshuaConfiguration.OOVItem;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+import joshua.corpus.Vocabulary;
+import joshua.decoder.chart_parser.SourcePath;
+
+/**
+ * This feature is fired when an out-of-vocabulary word (with respect to the translation model) is
+ * entered into the chart. OOVs work in the following manner: for each word in the input that is OOV
+ * with respect to the translation model, we create a rule that pushes that word through
+ * untranslated (the suffix "_OOV" can optionally be appended according to the runtime parameter
+ * "mark-oovs") . These rules are all stored in a grammar whose owner is "oov". The OOV feature
+ * function template then fires the "OOVPenalty" feature whenever it is asked to score an OOV rule.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+public class OOVPenalty extends StatelessFF {
+ private int ownerID = -1;
+
+ /* The default value returned for OOVs. Can be overridden with -oov-list */
+ private float defaultValue = -100f;
+ private HashMap<Integer,Float> oovWeights = null;
+
+ public OOVPenalty(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "OOVPenalty", args, config);
+
+ ownerID = Vocabulary.id("oov");
+ oovWeights = new HashMap<Integer,Float>();
+
+ if (config.oovList != null)
+ for (OOVItem item: config.oovList)
+ oovWeights.put(Vocabulary.id(item.label), item.weight);
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * OOV rules cover exactly one word, and such rules belong to a grammar whose owner is "oov". Each
+ * OOV fires the OOVPenalty feature with a value of 1, so the cost is simply the weight, which was
+ * cached when the feature was created.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && this.ownerID == rule.getOwner()) {
+// acc.add(name, getValue(rule.getLHS()));
+ acc.add(denseFeatureIndex, getValue(rule.getLHS()));
+ }
+
+ return null;
+ }
+
+ /**
+ * It's important for the OOV feature to contribute to the rule's estimated cost, so that OOV
+ * rules (which are added for all words, not just ones without translation options) get sorted
+ * to the bottom during cube pruning.
+ *
+ * Important! estimateCost returns the *weighted* feature value.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null && this.ownerID == rule.getOwner())
+ return weights.getDense(denseFeatureIndex) * getValue(rule.getLHS());
+ return 0.0f;
+ }
+
+ private float getValue(int lhs) {
+ return oovWeights.containsKey(lhs) ? oovWeights.get(lhs) : defaultValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java b/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
new file mode 100644
index 0000000..9882bc1
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import joshua.corpus.Vocabulary;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Grammar;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature handles the list of features that are found with grammar rules in the grammar file.
+ * dense features that may be associated with the rules in a grammar file. The feature names of
+ * these dense rules are a function of the phrase model owner. When the feature is loaded, it
+ * queries the weights for the set of features that are active for this grammar, storing them in an
+ * array.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Zhifei Li <zh...@gmail.com>
+ */
+
+public class PhraseModel extends StatelessFF {
+
+ /* The owner of the grammar. */
+ private int ownerID;
+ private String owner;
+
+ private float[] phrase_weights = null;
+
+ public PhraseModel(FeatureVector weights, String[] args, JoshuaConfiguration config, Grammar g) {
+ super(weights, "tm_", args, config);
+
+ String owner = parsedArgs.get("owner");
+ this.name = String.format("tm_%s", owner);
+
+ /*
+ * Determine the number of features by querying the example grammar that was passed in.
+ */
+ phrase_weights = new float[g.getNumDenseFeatures()];
+// System.err.println(String.format("GOT %d FEATURES FOR %s", g.getNumDenseFeatures(), owner));
+ for (int i = 0; i < phrase_weights.length; i++)
+ phrase_weights[i] = weights.getSparse(String.format("tm_%s_%d", owner, i));
+
+ // Store the owner.
+ this.owner = owner;
+ this.ownerID = Vocabulary.id(owner);
+ }
+
+ /**
+ * Just register a single weight, tm_OWNER, and use that to set its precomputed cost
+ */
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ for (int i = 0; i < phrase_weights.length; i++)
+ names.add(String.format("tm_%s_%d", owner, i));
+ return names;
+ }
+
+ /**
+ * Estimates the cost of applying this rule, which is just the score of the precomputable feature
+ * functions.
+ */
+ @Override
+ public float estimateCost(final Rule rule, Sentence sentence) {
+
+ if (rule != null && rule.getOwner() == ownerID) {
+ if (rule.getPrecomputableCost() <= Float.NEGATIVE_INFINITY)
+ rule.setPrecomputableCost(phrase_weights, weights);
+
+ return rule.getPrecomputableCost();
+ }
+
+ return 0.0f;
+ }
+
+ /**
+ * Just chain to computeFeatures(rule), since this feature doesn't use the sourcePath or sentID. *
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && rule.getOwner() == ownerID) {
+ /*
+ * Here, we peak at the Accumulator object. If it's asking for scores, then we don't bother to
+ * add each feature, but rather compute the inner product and add *that*. This is totally
+ * cheating; the Accumulator is supposed to be a generic object. But without this cheat
+ */
+ if (rule.getPrecomputableCost() <= Float.NEGATIVE_INFINITY) {
+ // float score = rule.getFeatureVector().innerProduct(weights);
+ rule.setPrecomputableCost(phrase_weights, weights);
+ }
+
+// System.err.println(String.format("RULE = %s / %f", rule.getEnglishWords(), rule.getPrecomputableCost()));
+ for (int k = 0; k < phrase_weights.length; k++) {
+// System.err.println(String.format("k = %d, denseFeatureIndex = %d, owner = %s, ownerID = %d", k, denseFeatureIndex, owner, ownerID));
+ acc.add(k + denseFeatureIndex, rule.getDenseFeature(k));
+ }
+
+ for (String key: rule.getFeatureVector().keySet())
+ acc.add(key, rule.getFeatureVector().getSparse(key));
+ }
+
+ return null;
+ }
+
+ public String toString() {
+ return name + " " + Vocabulary.word(ownerID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java b/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
new file mode 100644
index 0000000..fa6a3d1
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import joshua.corpus.Vocabulary;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.phrase.Hypothesis;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature just counts rules that are used. You can restrict it with a number of flags:
+ *
+ * -owner OWNER
+ * Only count rules owned by OWNER
+ * -target|-source
+ * Only count the target or source side (plus the LHS)
+ *
+ * TODO: add an option to separately provide a list of rule counts, restrict to counts above a threshold.
+ */
+public class PhrasePenalty extends StatelessFF {
+
+ private int owner = 0;
+ private float value = 1.0f;
+
+ public PhrasePenalty(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "PhrasePenalty", args, config);
+ if (parsedArgs.containsKey("owner"))
+ this.owner = Vocabulary.id(parsedArgs.get("owner"));
+ else // default
+ this.owner = Vocabulary.id("pt");
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE
+ && (owner == 0 || rule.getOwner() == owner))
+ acc.add(denseFeatureIndex, value);
+
+ return null;
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Returns the *weighted* estimate.
+ *
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null && rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE
+ && (owner == 0 || rule.getOwner() == owner))
+ return weights.getDense(denseFeatureIndex) * value;
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java b/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
new file mode 100644
index 0000000..cd7d9e7
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.List;
+
+import joshua.corpus.Vocabulary;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/*
+ * This feature computes a bin for the rule and activates a feature for it. It requires access to
+ * the index of the RarityPenalty field, from which the rule count can be computed.
+ */
+public class RuleCountBin extends StatelessFF {
+ private int field = -1;
+
+ public RuleCountBin(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleCountBin", args, config);
+
+ field = Integer.parseInt(parsedArgs.get("field"));
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule.getOwner() != Vocabulary.id("pt"))
+ return null;
+
+ float rarityPenalty = -rule.getFeatureVector().getSparse(String.format("tm_pt_%d", field));
+ int count = (int) (1.0 - Math.log(rarityPenalty));
+
+ String feature = "RuleCountBin_inf";
+
+ int[] bins = { 1, 2, 4, 8, 16, 32, 64, 128, 1000, 10000 };
+ for (int k : bins) {
+ if (count <= k) {
+ feature = String.format("RuleCountBin_%d", k);
+ break;
+ }
+ }
+
+ System.err.println(String.format("RuleCountBin(%f) = %d ==> %s", rarityPenalty, count, feature));
+
+ acc.add(feature, 1.0f);
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java b/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
new file mode 100644
index 0000000..9fb7d3e
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.List;
+
+import joshua.corpus.Vocabulary;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature just counts rules that are used. You can restrict it with a number of flags:
+ *
+ * -owner OWNER
+ * Only count rules owned by OWNER
+ * -target|-source
+ * Only count the target or source side (plus the LHS)
+ *
+ * TODO: add an option to separately provide a list of rule counts, restrict to counts above a threshold.
+ */
+public class RuleFF extends StatelessFF {
+
+ private enum Sides { SOURCE, TARGET, BOTH };
+
+ private int owner = 0;
+ private Sides sides = Sides.BOTH;
+
+ public RuleFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleFF", args, config);
+
+ owner = Vocabulary.id(parsedArgs.get("owner"));
+ if (parsedArgs.containsKey("source"))
+ sides = Sides.SOURCE;
+ else if (parsedArgs.containsKey("target"))
+ sides = Sides.TARGET;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (owner > 0 && rule.getOwner() == owner) {
+ String ruleString = getRuleString(rule);
+ acc.add(ruleString, 1);
+ }
+
+ return null;
+ }
+
+ private String getRuleString(Rule rule) {
+ String ruleString = "";
+ switch(sides) {
+ case BOTH:
+ ruleString = String.format("%s %s %s", Vocabulary.word(rule.getLHS()), rule.getFrenchWords(),
+ rule.getEnglishWords());
+ break;
+
+ case SOURCE:
+ ruleString = String.format("%s %s", Vocabulary.word(rule.getLHS()), rule.getFrenchWords());
+ break;
+
+ case TARGET:
+ ruleString = String.format("%s %s", Vocabulary.word(rule.getLHS()), rule.getEnglishWords());
+ break;
+ }
+ return ruleString.replaceAll("[ =]", "~");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java b/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java
new file mode 100644
index 0000000..645905a
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/*
+ * This feature computes three feature templates: a feature indicating the length of the rule's
+ * source side, its target side, and a feature that pairs them.
+ */
+public class RuleLength extends StatelessFF {
+
+ public RuleLength(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleLength", args, config);
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ int sourceLen = rule.getFrench().length;
+ int targetLen = rule.getEnglish().length;
+ acc.add(String.format("%s_sourceLength%d", name, sourceLen), 1);
+ acc.add(String.format("%s_targetLength%d", name, targetLen), 1);
+ acc.add(String.format("%s_pairLength%d-%d", name, sourceLen, targetLen), 1);
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java b/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java
new file mode 100644
index 0000000..777c790
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+import joshua.corpus.Vocabulary;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+
+public class RulePropertiesQuerying {
+
+ public static final String getLHSAsString(Rule rule) {
+ return Vocabulary.word(rule.getLHS());
+ }
+
+ public static List<String> getRuleSourceNonterminalStrings(Rule rule) {
+ List<String> result = new ArrayList<String>();
+ for (int nonTerminalIndex : rule.getForeignNonTerminals()) {
+ result.add(Vocabulary.word(nonTerminalIndex));
+ }
+ return result;
+ }
+
+ public static List<String> getSourceNonterminalStrings(List<HGNode> tailNodes) {
+ List<String> result = new ArrayList<String>();
+ for (HGNode tailNode : tailNodes) {
+ result.add(Vocabulary.word(tailNode.lhs));
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java b/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java
new file mode 100644
index 0000000..e243528
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/*
+ * Implements the RuleShape feature for source, target, and paired source+target sides.
+ */
+public class RuleShape extends StatelessFF {
+
+ public RuleShape(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleShape", args, config);
+ }
+
+ private int gettype(int id) {
+ if (id < 0)
+ return -1;
+ return 1;
+ }
+
+ private String pattern(int[] ids) {
+ StringBuilder pattern = new StringBuilder();
+ int curtype = gettype(ids[0]);
+ int curcount = 1;
+ for (int i = 1; i < ids.length; i++) {
+ if (gettype(ids[i]) != curtype) {
+ pattern.append(String.format("%s%s_", curtype < 0 ? "N" : "x", curcount > 1 ? "+" : ""));
+ curtype = gettype(ids[i]);
+ curcount = 1;
+ } else {
+ curcount++;
+ }
+ }
+ pattern.append(String.format("%s%s_", curtype < 0 ? "N" : "x", curcount > 1 ? "+" : ""));
+ return pattern.toString();
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i_, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ String sourceShape = pattern(rule.getFrench());
+ String targetShape = pattern(rule.getEnglish());
+ acc.add(String.format("%s_source_%s", name, sourceShape), 1);
+ acc.add(String.format("%s_target_%s", name, targetShape), 1);
+ acc.add(String.format("%s_both_%s__%s", name, sourceShape, targetShape), 1);
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java b/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java
new file mode 100644
index 0000000..2f490fa
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import joshua.decoder.segment_file.Sentence;
+
+public interface SourceDependentFF extends Cloneable {
+
+ public void setSource(Sentence sentence);
+
+ public FeatureFunction clone();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java b/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java
new file mode 100644
index 0000000..68dc595
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature returns the scored path through the source lattice, which is recorded in a
+ * SourcePath object.
+ *
+ * @author Chris Dyer <re...@umd.edu>
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+public final class SourcePathFF extends StatelessFF {
+
+ /*
+ * This is a single-value feature template, so we cache the weight here.
+ */
+ public SourcePathFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "SourcePath", args, config);
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ acc.add(denseFeatureIndex, sourcePath.getPathCost());
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java b/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
new file mode 100644
index 0000000..4ec2e57
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.List;
+
+import joshua.decoder.Decoder;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * Stateful features contribute dynamic programming state. Unlike earlier versions of Joshua, the
+ * stateful feature itself is responsible for computing and return its updated state. Each
+ * state-computing feature function is assigned a global index, which is used to index the list of
+ * state-contributing objects in each HGNode. State can no longer be shared among different feature
+ * functions.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Juri Ganitkevich <ju...@cs.jhu.edu>
+ */
+public abstract class StatefulFF extends FeatureFunction {
+
+ /* Every stateful FF takes a unique index value and increments this. */
+ static int GLOBAL_STATE_INDEX = 0;
+
+ /* This records the state index for each instantiated stateful feature function. */
+ protected int stateIndex = 0;
+
+ public StatefulFF(FeatureVector weights, String name, String[] args, JoshuaConfiguration config) {
+ super(weights, name, args, config);
+
+ Decoder.LOG(1, "Stateful object with state index " + GLOBAL_STATE_INDEX);
+ stateIndex = GLOBAL_STATE_INDEX++;
+ }
+
+ public static void resetGlobalStateIndex() {
+ GLOBAL_STATE_INDEX = 0;
+ }
+
+ public final boolean isStateful() {
+ return true;
+ }
+
+ public final int getStateIndex() {
+ return stateIndex;
+ }
+
+ /**
+ * Function computing the features that this function fires when a rule is applied. Must return
+ * its updated DPState. The accumulator is used to record every feature that fires.
+ */
+ @Override
+ public abstract DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc);
+
+ @Override
+ public abstract DPState computeFinal(HGNode tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc);
+
+ /**
+ * Computes an estimated future cost of this rule. Note that this is not compute as part of the
+ * score but is used for pruning.
+ */
+ @Override
+ public abstract float estimateFutureCost(Rule rule, DPState state, Sentence sentence);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java b/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java
new file mode 100644
index 0000000..198219b
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ * Stateless feature functions do not contribute any state. You need not implement this class to
+ * create a stateless feature function, but it provides a few convenience functions.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Juri Ganitkevich <ju...@cs.jhu.edu>
+ */
+
+public abstract class StatelessFF extends FeatureFunction {
+
+ public StatelessFF(FeatureVector weights, String name, String[] args, JoshuaConfiguration config) {
+ super(weights, name, args, config);
+ }
+
+ public final boolean isStateful() {
+ return false;
+ }
+
+ /**
+ * The estimated cost of applying this feature, given only the rule. This is used in sorting the
+ * rules for cube pruning. For most features, this will be 0.0.
+ */
+ public float estimateCost(Rule rule, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * Implementations of this should return null, since no state is contributed.
+ */
+ @Override
+ public abstract DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc);
+
+ /**
+ * Implementations of this should return null, since no state is contributed.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ return null;
+ }
+
+ /**
+ * Stateless functions do not have an estimate of the future cost because they do not have access
+ * to the state.
+ */
+ public final float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java b/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
new file mode 100644
index 0000000..846273d
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import joshua.corpus.Vocabulary;
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.state_maintenance.NgramDPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.segment_file.Sentence;
+import joshua.util.io.LineReader;
+
+/***
+ * The RuleBigram feature is an indicator feature that counts target word bigrams that are created when
+ * a rule is applied. It accepts three parameters:
+ *
+ * -vocab /path/to/vocab
+ *
+ * The path to a vocabulary, where each line is of the format ID WORD COUNT.
+ *
+ * -threshold N
+ *
+ * Mask to UNK all words whose COUNT is less than N.
+ *
+ * -top-n N
+ *
+ * Only use the top N words.
+ */
+
+public class TargetBigram extends StatefulFF {
+
+ private HashSet<String> vocab = null;
+ private int maxTerms = 1000000;
+ private int threshold = 0;
+
+ public TargetBigram(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "TargetBigram", args, config);
+
+ if (parsedArgs.containsKey("threshold"))
+ threshold = Integer.parseInt(parsedArgs.get("threshold"));
+
+ if (parsedArgs.containsKey("top-n"))
+ maxTerms = Integer.parseInt(parsedArgs.get("top-n"));
+
+ if (parsedArgs.containsKey("vocab")) {
+ loadVocab(parsedArgs.get("vocab"));
+ }
+ }
+
+ /**
+ * Load vocabulary items passing the 'threshold' and 'top-n' filters.
+ *
+ * @param filename
+ */
+ private void loadVocab(String filename) {
+ this.vocab = new HashSet<String>();
+ this.vocab.add("<s>");
+ this.vocab.add("</s>");
+ try {
+ LineReader lineReader = new LineReader(filename);
+ for (String line: lineReader) {
+ if (lineReader.lineno() > maxTerms)
+ break;
+
+ String[] tokens = line.split("\\s+");
+ String word = tokens[1];
+ int count = Integer.parseInt(tokens[2]);
+
+ if (count >= threshold)
+ vocab.add(word);
+ }
+
+ } catch (IOException e) {
+ System.err.println(String.format("* FATAL: couldn't load TargetBigram vocabulary '%s'", filename));
+ System.exit(1);
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int spanStart, int spanEnd,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc) {
+
+ int[] enWords = rule.getEnglish();
+
+ int left = -1;
+ int right = -1;
+
+ List<String> currentNgram = new LinkedList<String>();
+ for (int c = 0; c < enWords.length; c++) {
+ int curID = enWords[c];
+
+ if (Vocabulary.nt(curID)) {
+ int index = -(curID + 1);
+ NgramDPState state = (NgramDPState) tailNodes.get(index).getDPState(stateIndex);
+ int[] leftContext = state.getLeftLMStateWords();
+ int[] rightContext = state.getRightLMStateWords();
+
+ // Left context.
+ for (int token : leftContext) {
+ currentNgram.add(getWord(token));
+ if (left == -1)
+ left = token;
+ right = token;
+ if (currentNgram.size() == 2) {
+ String ngram = join(currentNgram);
+ acc.add(String.format("%s_%s", name, ngram), 1);
+// System.err.println(String.format("ADDING %s_%s", name, ngram));
+ currentNgram.remove(0);
+ }
+ }
+ // Replace right context.
+ int tSize = currentNgram.size();
+ for (int i = 0; i < rightContext.length; i++)
+ currentNgram.set(tSize - rightContext.length + i, getWord(rightContext[i]));
+
+ } else { // terminal words
+ currentNgram.add(getWord(curID));
+ if (left == -1)
+ left = curID;
+ right = curID;
+ if (currentNgram.size() == 2) {
+ String ngram = join(currentNgram);
+ acc.add(String.format("%s_%s", name, ngram), 1);
+// System.err.println(String.format("ADDING %s_%s", name, ngram));
+ currentNgram.remove(0);
+ }
+ }
+ }
+
+ NgramDPState state = new NgramDPState(new int[] { left }, new int[] { right });
+// System.err.println(String.format("RULE %s -> state %s", rule.getRuleString(), state));
+ return state;
+ }
+
+ /**
+ * Returns the word after comparing against the private vocabulary (if set).
+ *
+ * @param curID
+ * @return the word
+ */
+ private String getWord(int curID) {
+ String word = Vocabulary.word(curID);
+
+ if (vocab != null && ! vocab.contains(word)) {
+ return "UNK";
+ }
+
+ return word;
+ }
+
+ /**
+ * We don't compute a future cost.
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * There is nothing to be done here, since <s> and </s> are included in rules that are part
+ * of the grammar. We simply return the DP state of the tail node.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ return tailNode.getDPState(stateIndex);
+ }
+
+ /**
+ * TargetBigram features are only computed across hyperedges, so there is nothing to be done here.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * Join a list with the _ character. I am sure this is in a library somewhere.
+ *
+ * @param list a list of strings
+ * @return the joined String
+ */
+ private String join(List<String> list) {
+ StringBuilder sb = new StringBuilder();
+ for (String item : list) {
+ sb.append(item.toString() + "_");
+ }
+
+ return sb.substring(0, sb.length() - 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java b/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java
new file mode 100644
index 0000000..583b59c
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import joshua.decoder.JoshuaConfiguration;
+import joshua.decoder.ff.state_maintenance.DPState;
+import joshua.decoder.ff.tm.Rule;
+import joshua.decoder.chart_parser.SourcePath;
+import joshua.decoder.hypergraph.HGNode;
+import joshua.decoder.phrase.Hypothesis;
+import joshua.decoder.segment_file.Sentence;
+
+/**
+ *
+ * @author Zhifei Li <zh...@gmail.com>
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+public final class WordPenalty extends StatelessFF {
+
+ private float OMEGA = -(float) Math.log10(Math.E); // -0.435
+
+ public WordPenalty(final FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "WordPenalty", args, config);
+
+ if (parsedArgs.containsKey("value"))
+ OMEGA = Float.parseFloat(parsedArgs.get("value"));
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null) {
+ // TODO: this is an inefficient way to do this. Find a better way to not apply this rule
+ // to start and stop glue rules when phrase-based decoding.
+ if (config.search_algorithm.equals("cky")
+ || (rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE))
+ // acc.add(name, OMEGA * (rule.getEnglish().length - rule.getArity()));
+ acc.add(denseFeatureIndex, OMEGA * (rule.getEnglish().length - rule.getArity()));
+ }
+
+ return null;
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null)
+ return weights.getDense(denseFeatureIndex) * OMEGA * (rule.getEnglish().length - rule.getArity());
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java b/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java
new file mode 100644
index 0000000..b19d897
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.decoder.ff.fragmentlm;
+
+import java.util.*;
+
+/**
+ * Concatenates an iterator over iterators into one long iterator.
+ *
+ * @author Dan Klein
+ */
+public class ConcatenationIterator<E> implements Iterator<E> {
+
+ Iterator<Iterator<E>> sourceIterators;
+ Iterator<E> currentIterator;
+ Iterator<E> lastIteratorToReturn;
+
+ public boolean hasNext() {
+ if (currentIterator.hasNext())
+ return true;
+ return false;
+ }
+
+ public E next() {
+ if (currentIterator.hasNext()) {
+ E e = currentIterator.next();
+ lastIteratorToReturn = currentIterator;
+ advance();
+ return e;
+ }
+ throw new NoSuchElementException();
+ }
+
+ private void advance() {
+ while (! currentIterator.hasNext() && sourceIterators.hasNext()) {
+ currentIterator = sourceIterators.next();
+ }
+ }
+
+ public void remove() {
+ if (lastIteratorToReturn == null)
+ throw new IllegalStateException();
+ currentIterator.remove();
+ }
+
+ public ConcatenationIterator(Iterator<Iterator<E>> sourceIterators) {
+ this.sourceIterators = sourceIterators;
+ this.currentIterator = (new ArrayList<E>()).iterator();
+ this.lastIteratorToReturn = null;
+ advance();
+ }
+
+ public ConcatenationIterator(Collection<Iterator<E>> iteratorCollection) {
+ this(iteratorCollection.iterator());
+ }
+
+ public static void main(String[] args) {
+ List<String> list0 = Collections.emptyList();
+ List<String> list1 = Arrays.asList("a b c d".split(" "));
+ List<String> list2 = Arrays.asList("e f".split(" "));
+ List<Iterator<String>> iterators = new ArrayList<Iterator<String>>();
+ iterators.add(list1.iterator());
+ iterators.add(list0.iterator());
+ iterators.add(list2.iterator());
+ iterators.add(list0.iterator());
+ Iterator<String> iterator = new ConcatenationIterator<String>(iterators);
+ while (iterator.hasNext()) {
+ System.out.println(iterator.next());
+ }
+ }
+}