You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/08/30 21:04:48 UTC
[03/17] incubator-joshua git commit: Fix a number of issues: - Reader
now implements autocloseable - Close various leaks from LineReader -
LineReader no longer implements custom finalize(). Resources should be
explicitly closed when no longer needed. T
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java b/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
index f8173b8..0bda3e6 100644
--- a/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
@@ -21,27 +21,27 @@ package org.apache.joshua.decoder.ff.tm.packed;
/***
* This package implements Joshua's packed grammar structure, which enables the efficient loading
* and accessing of grammars. It is described in the paper:
- *
+ *
* @article{ganitkevitch2012joshua,
* Author = {Ganitkevitch, J. and Cao, Y. and Weese, J. and Post, M. and Callison-Burch, C.},
* Journal = {Proceedings of WMT12},
* Title = {Joshua 4.0: Packing, PRO, and paraphrases},
* Year = {2012}}
- *
+ *
* The packed grammar works by compiling out the grammar tries into a compact format that is loaded
* and parsed directly from Java arrays. A fundamental problem is that Java arrays are indexed
* by ints and not longs, meaning the maximum size of the packed grammar is about 2 GB. This forces
* the use of packed grammar slices, which together constitute the grammar. The figure in the
- * paper above shows what each slice looks like.
- *
+ * paper above shows what each slice looks like.
+ *
* The division across slices is done in a depth-first manner. Consider the entire grammar organized
* into a single source-side trie. The splits across tries are done by grouping the root-level
- * outgoing trie arcs --- and the entire trie beneath them --- across slices.
- *
- * This presents a problem: if the subtree rooted beneath a single top-level arc is too big for a
+ * outgoing trie arcs --- and the entire trie beneath them --- across slices.
+ *
+ * This presents a problem: if the subtree rooted beneath a single top-level arc is too big for a
* slice, the grammar can't be packed. This happens with very large Hiero grammars, for example,
* where there are a *lot* of rules that start with [X].
- *
+ *
* A solution being worked on is to split that symbol and pack them into separate grammars with a
* shared vocabulary, and then rely on Joshua's ability to query multiple grammars for rules to
* solve this problem. This is not currently implemented but could be done directly in the
@@ -63,7 +63,6 @@ import java.io.InputStream;
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
-import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Files;
@@ -73,7 +72,6 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -94,15 +92,14 @@ import org.apache.joshua.util.FormatUtils;
import org.apache.joshua.util.encoding.EncoderConfiguration;
import org.apache.joshua.util.encoding.FloatEncoder;
import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
public class PackedGrammar extends AbstractGrammar {
private static final Logger LOG = LoggerFactory.getLogger(PackedGrammar.class);
@@ -135,14 +132,14 @@ public class PackedGrammar extends AbstractGrammar {
if (!Vocabulary.read(vocabFile)) {
throw new RuntimeException("mismatches or collisions while reading on-disk vocabulary");
}
-
+
// Read the config
String configFile = grammar_dir + File.separator + "config";
if (new File(configFile).exists()) {
LOG.info("Reading packed config: {}", configFile);
readConfig(configFile);
}
-
+
// Read the quantizer setup.
LOG.info("Reading encoder configuration: {}{}encoding", grammar_dir, File.separator);
encoding = new EncoderConfiguration();
@@ -226,14 +223,14 @@ public class PackedGrammar extends AbstractGrammar {
* that represent the subtrie for a particular firstword.
* If the GrammarPacker has to distribute rules for a
* source-side firstword across multiple slices, a
- * SliceAggregatingTrie node is created that aggregates those
+ * SliceAggregatingTrie node is created that aggregates those
* tries to hide
* this additional complexity from the grammar interface
* This feature allows packing of grammars where the list of rules
* for a single source-side firstword would exceed the maximum array
* size of Java (2gb).
*/
- public final class PackedRoot implements Trie {
+ public static final class PackedRoot implements Trie {
private final HashMap<Integer, Trie> lookup;
@@ -241,9 +238,9 @@ public class PackedGrammar extends AbstractGrammar {
final Map<Integer, List<Trie>> childTries = collectChildTries(slices);
lookup = buildLookupTable(childTries);
}
-
+
/**
- * Determines whether trie nodes for source first-words are spread over
+ * Determines whether trie nodes for source first-words are spread over
* multiple packedSlices by counting their occurrences.
* @param slices
* @return A mapping from first word ids to a list of trie nodes.
@@ -251,12 +248,12 @@ public class PackedGrammar extends AbstractGrammar {
private Map<Integer, List<Trie>> collectChildTries(final List<PackedSlice> slices) {
final Map<Integer, List<Trie>> childTries = new HashMap<>();
for (PackedSlice packedSlice : slices) {
-
+
// number of tries stored in this packedSlice
final int num_children = packedSlice.source[0];
for (int i = 0; i < num_children; i++) {
final int id = packedSlice.source[2 * i + 1];
-
+
/* aggregate tries with same root id
* obtain a Trie node, already at the correct address in the packedSlice.
* In other words, the lookup index already points to the correct trie node in the packedSlice.
@@ -271,7 +268,7 @@ public class PackedGrammar extends AbstractGrammar {
}
return childTries;
}
-
+
/**
* Build a lookup table for children tries.
* If the list contains only a single child node, a regular trie node
@@ -497,7 +494,7 @@ public class PackedGrammar extends AbstractGrammar {
}
featurePosition += EncoderConfiguration.ID_SIZE + encoder.size();
}
-
+
return featureVector;
}
@@ -511,7 +508,7 @@ public class PackedGrammar extends AbstractGrammar {
if (alignments == null)
throw new RuntimeException("No alignments available.");
int alignment_position = getIntFromByteBuffer(block_id, alignments);
- int num_points = (int) alignments.get(alignment_position);
+ int num_points = alignments.get(alignment_position);
byte[] alignment = new byte[num_points * 2];
alignments.position(alignment_position + 1);
@@ -533,6 +530,7 @@ public class PackedGrammar extends AbstractGrammar {
return getTrie(0);
}
+ @Override
public String toString() {
return name;
}
@@ -540,9 +538,9 @@ public class PackedGrammar extends AbstractGrammar {
/**
* A trie node within the grammar slice. Identified by its position within the source array,
* and, as a supplement, the source string leading from the trie root to the node.
- *
+ *
* @author jg
- *
+ *
*/
public class PackedTrie implements Trie, RuleCollection {
@@ -785,14 +783,14 @@ public class PackedGrammar extends AbstractGrammar {
throw new UnsupportedOperationException();
}
}
-
+
/**
* A packed phrase pair represents a rule of the form of a phrase pair, packed with the
* grammar-packer.pl script, which simply adds a nonterminal [X] to the left-hand side of
* all phrase pairs (and converts the Moses features). The packer then packs these. We have
* to then put a nonterminal on the source and target sides to treat the phrase pairs like
- * left-branching rules, which is how Joshua deals with phrase decoding.
- *
+ * left-branching rules, which is how Joshua deals with phrase decoding.
+ *
* @author Matt Post post@cs.jhu.edu
*
*/
@@ -845,17 +843,17 @@ public class PackedGrammar extends AbstractGrammar {
/**
* Take the English phrase of the underlying rule and prepend an [X].
- *
+ *
* @return the augmented phrase
*/
@Override
public int[] getEnglish() {
return this.englishSupplier.get();
}
-
+
/**
* Take the French phrase of the underlying rule and prepend an [X].
- *
+ *
* @return the augmented French phrase
*/
@Override
@@ -866,10 +864,10 @@ public class PackedGrammar extends AbstractGrammar {
System.arraycopy(src, 0, phrase, 1, src.length);
return phrase;
}
-
+
/**
* Similarly the alignment array needs to be shifted over by one.
- *
+ *
* @return the byte[] alignment
*/
@Override
@@ -967,12 +965,12 @@ public class PackedGrammar extends AbstractGrammar {
public FeatureVector getFeatureVector() {
return this.featureVectorSupplier.get();
}
-
+
@Override
public byte[] getAlignment() {
return this.alignmentsSupplier.get();
}
-
+
@Override
public String getAlignmentString() {
throw new RuntimeException("AlignmentString not implemented for PackedRule!");
@@ -1018,23 +1016,23 @@ public class PackedGrammar extends AbstractGrammar {
public void addOOVRules(int word, List<FeatureFunction> featureFunctions) {
throw new RuntimeException("PackedGrammar.addOOVRules(): I can't add OOV rules");
}
-
+
@Override
public void addRule(Rule rule) {
throw new RuntimeException("PackedGrammar.addRule(): I can't add rules");
}
-
- /**
+
+ /**
* Read the config file
- *
+ *
* TODO: this should be rewritten using typeconfig.
- *
+ *
* @param config
* @throws IOException
*/
private void readConfig(String config) throws IOException {
int version = 0;
-
+
for (String line: new LineReader(config)) {
String[] tokens = line.split(" = ");
if (tokens[0].equals("max-source-len"))
@@ -1043,7 +1041,7 @@ public class PackedGrammar extends AbstractGrammar {
version = Integer.parseInt(tokens[1]);
}
}
-
+
if (version != 3) {
String message = String.format("The grammar at %s was packed with packer version %d, but the earliest supported version is %d",
this.grammarDir, version, SUPPORTED_VERSION);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/lattice/Lattice.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/lattice/Lattice.java b/src/main/java/org/apache/joshua/lattice/Lattice.java
index 2332159..c557c07 100644
--- a/src/main/java/org/apache/joshua/lattice/Lattice.java
+++ b/src/main/java/org/apache/joshua/lattice/Lattice.java
@@ -24,7 +24,6 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -361,6 +360,7 @@ public class Lattice<Value> implements Iterable<Node<Value>> {
*
* @return An iterator over the nodes in this lattice.
*/
+ @Override
public Iterator<Node<Value>> iterator() {
return nodes.iterator();
}
@@ -471,56 +471,10 @@ public class Lattice<Value> implements Iterable<Node<Value>> {
}
/**
- * Topologically sorts the nodes and reassigns their numbers. Assumes that the first node is the
- * source, but otherwise assumes nothing about the input.
- *
- * Probably correct, but untested.
- */
- @SuppressWarnings("unused")
- private void topologicalSort() {
- HashMap<Node<Value>, List<Arc<Value>>> outgraph = new HashMap<Node<Value>, List<Arc<Value>>>();
- HashMap<Node<Value>, List<Arc<Value>>> ingraph = new HashMap<Node<Value>, List<Arc<Value>>>();
- for (Node<Value> node: nodes) {
- ArrayList<Arc<Value>> arcs = new ArrayList<Arc<Value>>();
- for (Arc<Value> arc: node.getOutgoingArcs()) {
- arcs.add(arc);
-
- if (! ingraph.containsKey(arc.getHead()))
- ingraph.put(arc.getHead(), new ArrayList<Arc<Value>>());
- ingraph.get(arc.getHead()).add(arc);
-
- outgraph.put(node, arcs);
- }
- }
-
- ArrayList<Node<Value>> sortedNodes = new ArrayList<Node<Value>>();
- Stack<Node<Value>> stack = new Stack<Node<Value>>();
- stack.push(nodes.get(0));
-
- while (! stack.empty()) {
- Node<Value> node = stack.pop();
- sortedNodes.add(node);
- for (Arc<Value> arc: outgraph.get(node)) {
- outgraph.get(node).remove(arc);
- ingraph.get(arc.getHead()).remove(arc);
-
- if (ingraph.get(arc.getHead()).size() == 0)
- sortedNodes.add(arc.getHead());
- }
- }
-
- int id = 0;
- for (Node<Value> node : sortedNodes)
- node.setID(id++);
-
- this.nodes = sortedNodes;
- }
-
- /**
- * Constructs a lattice from a given string representation.
+ * Constructs a lattice from a given string representation.
*
- * @param data String representation of a lattice.
- * @return A lattice that corresponds to the given string.
+ * @param data String representation of a lattice.
+ * @return A lattice that corresponds to the given string.
*/
public static Lattice<String> createFromString(String data) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/metrics/CHRF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/metrics/CHRF.java b/src/main/java/org/apache/joshua/metrics/CHRF.java
index d67f6e0..dcf606a 100644
--- a/src/main/java/org/apache/joshua/metrics/CHRF.java
+++ b/src/main/java/org/apache/joshua/metrics/CHRF.java
@@ -27,18 +27,18 @@ import java.util.logging.Logger;
* - Adapted to extend Joshua's EvaluationMetric class
* - Use of a length penalty to prevent chrF to prefer too long (with beta %gt; 1) or too short (with beta < 1) translations
* - Use of hash tables for efficient n-gram matching
- *
+ *
* The metric has 2 parameters:
* - Beta. It assigns beta times more weight to recall than to precision. By default 1.
* Although for evaluation the best correlation was found with beta=3, we've found the
* best results for tuning so far with beta=1
* - Max-ngram. Maximum n-gram length (characters). By default 6.
- *
+ *
* If you use this metric in your research please cite [2].
- *
+ *
* [1] Maja Popovic. 2015. chrF: character n-gram F-score for automatic MT evaluation.
* In Proceedings of the Tenth Workshop on Statistical Machine Translation. Lisbon, Portugal, pages 392\u2013395.
- * [2] V�ctor S�nchez Cartagena and Antonio Toral. 2016.
+ * [2] V�ctor S�nchez Cartagena and Antonio Toral. 2016.
* Abu-MaTran at WMT 2016 Translation Task: Deep Learning, Morphological Segmentation and Tuning on Character Sequences.
* In Proceedings of the First Conference on Machine Translation (WMT16). Berlin, Germany.
@@ -51,17 +51,17 @@ public class CHRF extends EvaluationMetric {
protected double factor;
protected int maxGramLength = 6; // The maximum n-gram we care about
//private double[] nGramWeights; //TODO to weight them differently
-
+
//private String metricName;
//private boolean toBeMinimized;
//private int suffStatsCount;
-
+
public CHRF()
{
this(1, 6);
}
-
+
public CHRF(String[] CHRF_options)
{
//
@@ -71,44 +71,47 @@ public class CHRF extends EvaluationMetric {
//
this(Double.parseDouble(CHRF_options[0]), Integer.parseInt(CHRF_options[1]));
}
-
- public CHRF(double bt, int mxGrmLn){
+
+ public CHRF(double bt, int mxGrmLn){
if (bt > 0) {
beta = bt;
} else {
logger.severe("Beta must be positive");
System.exit(1);
}
-
+
if (mxGrmLn >= 1) {
maxGramLength = mxGrmLn;
} else {
logger.severe("Maximum gram length must be positive");
System.exit(1);
}
-
+
initialize(); // set the data members of the metric
}
+ @Override
protected void initialize()
{
metricName = "CHRF";
toBeMinimized = false;
suffStatsCount = 4 * maxGramLength;
- factor = Math.pow(beta, 2);
+ factor = Math.pow(beta, 2);
}
-
+
+ @Override
public double bestPossibleScore() { return 100.0; }
-
+
+ @Override
public double worstPossibleScore() { return 0.0; }
protected String separateCharacters(String s)
{
- String s_chars = "";
+ String s_chars = "";
//alternative implementation (the one below seems more robust)
/*for (int i = 0; i < s.length(); i++) {
if (s.charAt(i) == ' ') continue;
- s_chars += s.charAt(i) + " ";
+ s_chars += s.charAt(i) + " ";
}
System.out.println("CHRF separate chars1: " + s_chars);*/
@@ -122,7 +125,7 @@ public class CHRF extends EvaluationMetric {
return s_chars;
}
-
+
protected HashMap<String, Integer>[] getGrams(String s)
{
HashMap<String, Integer>[] grams = new HashMap[1 + maxGramLength];
@@ -131,7 +134,7 @@ public class CHRF extends EvaluationMetric {
grams[n] = new HashMap<String, Integer>();
}
-
+
for (int n=1; n<=maxGramLength; n++){
String gram = "";
for (int i = 0; i < s.length() - n + 1; i++){
@@ -143,7 +146,7 @@ public class CHRF extends EvaluationMetric {
grams[n].put(gram, 1);
}
}
-
+
}
/* debugging
@@ -153,25 +156,22 @@ public class CHRF extends EvaluationMetric {
for (String gram: grams[n].keySet()){
key = gram.toString();
value = grams[n].get(gram).toString();
- System.out.println(key + " " + value);
+ System.out.println(key + " " + value);
}
}*/
-
+
return grams;
}
-
+
protected int[] candRefErrors(HashMap<String, Integer> ref, HashMap<String, Integer> cand)
{
int[] to_return = {0,0};
String gram;
- int cand_grams = 0, ref_grams = 0;
+ int cand_grams = 0;
int candGramCount = 0, refGramCount = 0;
int errors = 0;
- double result = 0;
- String not_found = "";
-
-
+
Iterator<String> it = (cand.keySet()).iterator();
while (it.hasNext()) {
@@ -180,41 +180,36 @@ public class CHRF extends EvaluationMetric {
cand_grams += candGramCount;
if (ref.containsKey(gram)) {
refGramCount = ref.get(gram);
- ref_grams += refGramCount;
if(candGramCount>refGramCount){
int error_here = candGramCount - refGramCount;
errors += error_here;
- not_found += gram + " (" + error_here + " times) ";
}
} else {
refGramCount = 0;
errors += candGramCount;
- not_found += gram + " ";
- }
+ }
}
-
+
//System.out.println(" Ngrams not found: " + not_found);
-
+
to_return[0] = cand_grams;
to_return[1] = errors;
-
+
return to_return;
}
-
+
+ @Override
public int[] suffStats(String cand_str, int i) //throws Exception
{
int[] stats = new int[suffStatsCount];
- double[] precisions = new double[maxGramLength+1];
- double[] recalls = new double[maxGramLength+1];
-
//TODO check unicode chars correctly split
String cand_char = separateCharacters(cand_str);
String ref_char = separateCharacters(refSentences[i][0]);
-
+
HashMap<String, Integer>[] grams_cand = getGrams(cand_char);
HashMap<String, Integer>[] grams_ref = getGrams(ref_char);
-
+
for (int n = 1; n <= maxGramLength; ++n) {
//System.out.println("Calculating precision...");
int[] precision_vals = candRefErrors(grams_ref[n], grams_cand[n]);
@@ -222,7 +217,7 @@ public class CHRF extends EvaluationMetric {
//System.out.println("Calculating recall...");
int[] recall_vals = candRefErrors(grams_cand[n], grams_ref[n]);
//System.out.println(" length: " + recall_vals[0] + ", errors: " + recall_vals[1]);
-
+
stats[4*(n-1)] = precision_vals[0]; //cand_grams
stats[4*(n-1)+1] = precision_vals[1]; //errors (precision)
stats[4*(n-1)+2] = recall_vals[0]; //ref_grams
@@ -233,6 +228,7 @@ public class CHRF extends EvaluationMetric {
}
+ @Override
public double score(int[] stats)
{
int precision_ngrams, recall_ngrams, precision_errors, recall_errors;
@@ -240,9 +236,9 @@ public class CHRF extends EvaluationMetric {
double[] recalls = new double[maxGramLength+1];
double[] fs = new double[maxGramLength+1];
//double[] scs = new double[maxGramLength+1];
- double totalPrecision = 0, totalRecall = 0, totalF = 0, totalSC = 0;
+ double totalF = 0, totalSC = 0;
double lp = 1;
-
+
if (stats.length != suffStatsCount) {
System.out.println("Mismatch between stats.length and suffStatsCount (" + stats.length + " vs. " + suffStatsCount + ") in NewMetric.score(int[])");
System.exit(1);
@@ -257,42 +253,41 @@ public class CHRF extends EvaluationMetric {
if (precision_ngrams != 0)
precisions[n] = 100 - 100*precision_errors/ (double)precision_ngrams;
else precisions[n] = 0;
-
+
if (recall_ngrams != 0)
recalls[n] = 100 - 100*recall_errors/ (double)recall_ngrams;
else
recalls[n] = 0;
-
+
if(precisions[n] != 0 || recalls[n] != 0)
fs[n] = (1+factor) * recalls[n] * precisions[n] / (factor * precisions[n] + recalls[n]);
else
fs[n] = 0;
-
+
//System.out.println("Precision (n=" + n + "): " + precisions[n]);
//System.out.println("Recall (n=" + n + "): " + recalls[n]);
//System.out.println("F (n=" + n + "): " + fs[n]);
- totalPrecision += (1/(double)maxGramLength) * precisions[n];
- totalRecall += (1/(double)maxGramLength) * recalls[n];
totalF += (1/(double)maxGramLength) * fs[n];
}
//length penalty
- if (beta>1){ //penalise long translations
+ if (beta>1){ //penalise long translations
lp = Math.min(1, stats[2]/(double)stats[0]);
} else if (beta < 1){ //penalise short translations
lp = Math.min(1, stats[0]/(double)stats[2]);
}
totalSC = totalF*lp;
-
+
//System.out.println("Precision (total): " + totalPrecision);
//System.out.println("Recall (total):" + totalRecall);
//System.out.println("F (total): " + totalF);
-
+
return totalSC;
}
+ @Override
public void printDetailedScore_fromStats(int[] stats, boolean oneLiner)
{
System.out.println(metricName + " = " + score(stats));
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/metrics/SARI.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/metrics/SARI.java b/src/main/java/org/apache/joshua/metrics/SARI.java
index 129e4af..9ee3af3 100644
--- a/src/main/java/org/apache/joshua/metrics/SARI.java
+++ b/src/main/java/org/apache/joshua/metrics/SARI.java
@@ -14,30 +14,30 @@
*/
package org.apache.joshua.metrics;
-// Changed PROCore.java (text normalization function) and EvaluationMetric too
-
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.logging.Logger;
-
import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Iterator;
+
+// Changed PROCore.java (text normalization function) and EvaluationMetric too
+
+import java.util.Map;
+import java.util.logging.Logger;
/***
* Implementation of the SARI metric for text-to-text correction.
- *
+ *
* \@article{xu2016optimizing,
* title={Optimizing statistical machine translation for text simplification},
* author={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},
* journal={Transactions of the Association for Computational Linguistics},
* volume={4},
* year={2016}}
- *
+ *
* @author Wei Xu
*/
public class SARI extends EvaluationMetric {
@@ -77,6 +77,7 @@ public class SARI extends EvaluationMetric {
}
+ @Override
protected void initialize() {
metricName = "SARI";
toBeMinimized = false;
@@ -88,10 +89,12 @@ public class SARI extends EvaluationMetric {
}
+ @Override
public double bestPossibleScore() {
return 1.0;
}
+ @Override
public double worstPossibleScore() {
return 0.0;
}
@@ -165,6 +168,7 @@ public class SARI extends EvaluationMetric {
}
// set contents of stats[] here!
+ @Override
public int[] suffStats(String cand_str, int i) {
int[] stats = new int[suffStatsCount];
@@ -173,9 +177,9 @@ public class SARI extends EvaluationMetric {
for (int n = 1; n <= maxGramLength; ++n) {
// ADD OPERATIONS
- HashMap cand_sub_src = substractHashMap(candNgramCounts[n], srcNgramCounts[i][n]);
- HashMap cand_and_ref_sub_src = intersectHashMap(cand_sub_src, refNgramCounts[i][n]);
- HashMap ref_sub_src = substractHashMap(refNgramCounts[i][n], srcNgramCounts[i][n]);
+ HashMap<String, Integer> cand_sub_src = substractHashMap(candNgramCounts[n], srcNgramCounts[i][n]);
+ HashMap<String, Integer> cand_and_ref_sub_src = intersectHashMap(cand_sub_src, refNgramCounts[i][n]);
+ HashMap<String, Integer> ref_sub_src = substractHashMap(refNgramCounts[i][n], srcNgramCounts[i][n]);
stats[StatIndex.values().length * (n - 1)
+ StatIndex.ADDBOTH.ordinal()] = cand_and_ref_sub_src.keySet().size();
@@ -190,11 +194,11 @@ public class SARI extends EvaluationMetric {
// System.out.println("ref_sub_src" + ref_sub_src + ref_sub_src.keySet().size());
// DELETION OPERATIONS
- HashMap src_sub_cand = substractHashMap(srcNgramCounts[i][n], candNgramCounts[n],
- this.refsPerSen, this.refsPerSen);
- HashMap src_sub_ref = substractHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
- this.refsPerSen, 1);
- HashMap src_sub_cand_sub_ref = intersectHashMap(src_sub_cand, src_sub_ref, 1, 1);
+ HashMap<String, Integer> src_sub_cand = substractHashMap(srcNgramCounts[i][n], candNgramCounts[n],
+ refsPerSen, refsPerSen);
+ HashMap<String, Integer> src_sub_ref = substractHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
+ refsPerSen, 1);
+ HashMap<String, Integer> src_sub_cand_sub_ref = intersectHashMap(src_sub_cand, src_sub_ref, 1, 1);
stats[StatIndex.values().length * (n - 1) + StatIndex.DELBOTH.ordinal()] = sumHashMapByValues(
src_sub_cand_sub_ref);
@@ -209,14 +213,14 @@ public class SARI extends EvaluationMetric {
// System.out.println("src_sub_ref" + src_sub_ref + sumHashMapByValues(src_sub_ref));
stats[StatIndex.values().length * (n - 1) + StatIndex.DELREF.ordinal()] = src_sub_ref.keySet()
- .size() * this.refsPerSen;
+ .size() * refsPerSen;
// KEEP OPERATIONS
- HashMap src_and_cand = intersectHashMap(srcNgramCounts[i][n], candNgramCounts[n],
- this.refsPerSen, this.refsPerSen);
- HashMap src_and_ref = intersectHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
- this.refsPerSen, 1);
- HashMap src_and_cand_and_ref = intersectHashMap(src_and_cand, src_and_ref, 1, 1);
+ HashMap<String, Integer> src_and_cand = intersectHashMap(srcNgramCounts[i][n], candNgramCounts[n],
+ refsPerSen, refsPerSen);
+ HashMap<String, Integer> src_and_ref = intersectHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
+ refsPerSen, 1);
+ HashMap<String, Integer> src_and_cand_and_ref = intersectHashMap(src_and_cand, src_and_ref, 1, 1);
stats[StatIndex.values().length * (n - 1)
+ StatIndex.KEEPBOTH.ordinal()] = sumHashMapByValues(src_and_cand_and_ref);
@@ -232,55 +236,11 @@ public class SARI extends EvaluationMetric {
divideHashMap(src_and_cand_and_ref, src_and_ref));
stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPREF.ordinal()] = src_and_ref
.keySet().size();
-
- // System.out.println("src_and_cand_and_ref" + src_and_cand_and_ref);
- // System.out.println("src_and_cand" + src_and_cand);
- // System.out.println("src_and_ref" + src_and_ref);
-
- // stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPBOTH2.ordinal()] = (int)
- // sumHashMapByDoubleValues(divideHashMap(src_and_cand_and_ref,src_and_ref)) * 100000000 /
- // src_and_ref.keySet().size() ;
- // stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPREF.ordinal()] =
- // src_and_ref.keySet().size() * 8;
-
- // System.out.println("src_and_cand_and_ref" + src_and_cand_and_ref);
- // System.out.println("src_and_cand" + src_and_cand);
- // System.out.println("divide" + divideHashMap(src_and_cand_and_ref,src_and_cand));
- // System.out.println(sumHashMapByDoubleValues(divideHashMap(src_and_cand_and_ref,src_and_cand)));
-
}
-
- int n = 1;
-
- // System.out.println("CAND: " + candNgramCounts[n]);
- // System.out.println("SRC: " + srcNgramCounts[i][n]);
- // System.out.println("REF: " + refNgramCounts[i][n]);
-
- HashMap src_and_cand = intersectHashMap(srcNgramCounts[i][n], candNgramCounts[n],
- this.refsPerSen, this.refsPerSen);
- HashMap src_and_ref = intersectHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
- this.refsPerSen, 1);
- HashMap src_and_cand_and_ref = intersectHashMap(src_and_cand, src_and_ref, 1, 1);
- // System.out.println("SRC&CAND&REF : " + src_and_cand_and_ref);
-
- HashMap cand_sub_src = substractHashMap(candNgramCounts[n], srcNgramCounts[i][n]);
- HashMap cand_and_ref_sub_src = intersectHashMap(cand_sub_src, refNgramCounts[i][n]);
- // System.out.println("CAND&REF-SRC : " + cand_and_ref_sub_src);
-
- HashMap src_sub_cand = substractHashMap(srcNgramCounts[i][n], candNgramCounts[n],
- this.refsPerSen, this.refsPerSen);
- HashMap src_sub_ref = substractHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
- this.refsPerSen, 1);
- HashMap src_sub_cand_sub_ref = intersectHashMap(src_sub_cand, src_sub_ref, 1, 1);
- // System.out.println("SRC-REF-CAND : " + src_sub_cand_sub_ref);
-
- // System.out.println("DEBUG:" + Arrays.toString(stats));
- // System.out.println("REF-SRC: " + substractHashMap(refNgramCounts[i], srcNgramCounts[i][0],
- // (double)refsPerSen));
-
return stats;
}
+ @Override
public double score(int[] stats) {
if (stats.length != suffStatsCount) {
System.out.println("Mismatch between stats.length and suffStatsCount (" + stats.length
@@ -320,23 +280,13 @@ public class SARI extends EvaluationMetric {
+ StatIndex.DELBOTH.ordinal()];
int delCandTotalNgram = stats[StatIndex.values().length * (n - 1)
+ StatIndex.DELCAND.ordinal()];
- int delRefTotalNgram = stats[StatIndex.values().length * (n - 1)
- + StatIndex.DELREF.ordinal()];
double prec_del_n = 0.0;
if (delCandTotalNgram > 0) {
prec_del_n = delCandCorrectNgram / (double) delCandTotalNgram;
}
- double recall_del_n = 0.0;
- if (delRefTotalNgram > 0) {
- recall_del_n = delCandCorrectNgram / (double) delRefTotalNgram;
- }
-
// System.out.println("\nDEBUG-SARI:" + delCandCorrectNgram + " " + delRefTotalNgram);
-
- double f1_del_n = meanHarmonic(prec_del_n, recall_del_n);
-
// sc += weights[n] * f1_del_n;
sc += weights[n] * prec_del_n;
@@ -410,7 +360,7 @@ public class SARI extends EvaluationMetric {
double sumcounts = 0;
for (Map.Entry<String, Double> e : counter.entrySet()) {
- sumcounts += (double) e.getValue();
+ sumcounts += e.getValue();
}
return sumcounts;
@@ -420,7 +370,7 @@ public class SARI extends EvaluationMetric {
int sumcounts = 0;
for (Map.Entry<String, Integer> e : counter.entrySet()) {
- sumcounts += (int) e.getValue();
+ sumcounts += e.getValue();
}
return sumcounts;
@@ -432,7 +382,6 @@ public class SARI extends EvaluationMetric {
for (Map.Entry<String, Integer> e : counter1.entrySet()) {
String ngram = e.getKey();
- int count1 = e.getValue();
int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
if (count2 == 0) {
newcounter.put(ngram, 1);
@@ -482,7 +431,6 @@ public class SARI extends EvaluationMetric {
for (Map.Entry<String, Integer> e : counter1.entrySet()) {
String ngram = e.getKey();
- int count1 = e.getValue();
int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
if (count2 > 0) {
newcounter.put(ngram, 1);
@@ -661,6 +609,7 @@ public class SARI extends EvaluationMetric {
}
+ @Override
public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
System.out.println(metricName + " = " + score(stats));
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/mira/MIRACore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/mira/MIRACore.java b/src/main/java/org/apache/joshua/mira/MIRACore.java
index 42dd995..a4a6b84 100755
--- a/src/main/java/org/apache/joshua/mira/MIRACore.java
+++ b/src/main/java/org/apache/joshua/mira/MIRACore.java
@@ -44,11 +44,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
+import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.metrics.EvaluationMetric;
import org.apache.joshua.util.StreamGobbler;
-import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.util.io.ExistingUTF8EncodedTextFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,19 +65,15 @@ public class MIRACore {
private TreeSet<Integer>[] indicesOfInterest_all;
private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
- private final Runtime myRuntime = Runtime.getRuntime();
private final static double NegInf = (-1.0 / 0.0);
private final static double PosInf = (+1.0 / 0.0);
private final static double epsilon = 1.0 / 1000000;
- private int progress;
-
private int verbosity; // anything of priority <= verbosity will be printed
// (lower value for priority means more important)
private Random randGen;
- private int generatedRands;
private int numSentences;
// number of sentences in the dev set
@@ -235,7 +232,7 @@ public class MIRACore {
private boolean usePseudoBleu = true; // need to use pseudo corpus to compute bleu?
private boolean returnBest = false; // return the best weight during tuning
private boolean needScale = true; // need scaling?
- private String trainingMode;
+
private int oraSelectMode = 1;
private int predSelectMode = 1;
private int miraIter = 1;
@@ -263,28 +260,27 @@ public class MIRACore {
this.joshuaConfiguration = joshuaConfiguration;
}
- public MIRACore(String[] args, JoshuaConfiguration joshuaConfiguration) {
+ public MIRACore(String[] args, JoshuaConfiguration joshuaConfiguration) throws FileNotFoundException, IOException {
this.joshuaConfiguration = joshuaConfiguration;
EvaluationMetric.set_knownMetrics();
processArgsArray(args);
initialize(0);
}
- public MIRACore(String configFileName, JoshuaConfiguration joshuaConfiguration) {
+ public MIRACore(String configFileName, JoshuaConfiguration joshuaConfiguration) throws FileNotFoundException, IOException {
this.joshuaConfiguration = joshuaConfiguration;
EvaluationMetric.set_knownMetrics();
processArgsArray(cfgFileToArgsArray(configFileName));
initialize(0);
}
- private void initialize(int randsToSkip) {
+ private void initialize(int randsToSkip) throws FileNotFoundException, IOException {
println("NegInf: " + NegInf + ", PosInf: " + PosInf + ", epsilon: " + epsilon, 4);
randGen = new Random(seed);
for (int r = 1; r <= randsToSkip; ++r) {
randGen.nextDouble();
}
- generatedRands = randsToSkip;
if (randsToSkip == 0) {
println("----------------------------------------------------", 1);
@@ -298,7 +294,7 @@ public class MIRACore {
// count the total num of sentences to be decoded, reffilename is the combined reference file
// name(auto generated)
- numSentences = countLines(refFileName) / refsPerSen;
+ numSentences = new ExistingUTF8EncodedTextFile(refFileName).getNumberOfLines() / refsPerSen;
// ??
processDocInfo();
@@ -311,7 +307,7 @@ public class MIRACore {
set_docSubsetInfo(docSubsetInfo);
// count the number of initial features
- numParams = countNonEmptyLines(paramsFileName) - 1;
+ numParams = new ExistingUTF8EncodedTextFile(paramsFileName).getNumberOfNonEmptyLines() - 1;
numParamsOld = numParams;
// read parameter config file
@@ -862,7 +858,6 @@ public class MIRACore {
// iterations if the user specifies a value for prevMERTIterations
// that causes MERT to skip candidates from early iterations.
- double[] currFeatVal = new double[1 + numParams];
String[] featVal_str;
int totalCandidateCount = 0;
@@ -1105,7 +1100,6 @@ public class MIRACore {
for (String featurePair : featVal_str) {
String[] pair = featurePair.split("=");
String name = pair[0];
- Double value = Double.parseDouble(pair[1]);
int featId = Vocabulary.id(name);
// need to identify newly fired feats here
@@ -1529,7 +1523,7 @@ public class MIRACore {
/*
* line format:
- *
+ *
* i ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val
* .*
*/
@@ -1599,8 +1593,6 @@ public class MIRACore {
BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
PrintWriter outFile = new PrintWriter(cfgFileName);
- BufferedReader inFeatDefFile = null;
- PrintWriter outFeatDefFile = null;
int origFeatNum = 0; // feat num in the template file
String line = inFile.readLine();
@@ -1803,7 +1795,7 @@ public class MIRACore {
// belongs to,
// and its order in that document. (can also use '-' instead of '_')
- int docInfoSize = countNonEmptyLines(docInfoFileName);
+ int docInfoSize = new ExistingUTF8EncodedTextFile(docInfoFileName).getNumberOfNonEmptyLines();
if (docInfoSize < numSentences) { // format #1 or #2
numDocuments = docInfoSize;
@@ -1913,13 +1905,13 @@ public class MIRACore {
/*
* InputStream inStream = new FileInputStream(new File(origFileName)); BufferedReader inFile =
* new BufferedReader(new InputStreamReader(inStream, "utf8"));
- *
+ *
* FileOutputStream outStream = new FileOutputStream(newFileName, false); OutputStreamWriter
* outStreamWriter = new OutputStreamWriter(outStream, "utf8"); BufferedWriter outFile = new
* BufferedWriter(outStreamWriter);
- *
+ *
* String line; while(inFile.ready()) { line = inFile.readLine(); writeLine(line, outFile); }
- *
+ *
* inFile.close(); outFile.close();
*/
return true;
@@ -2017,10 +2009,10 @@ public class MIRACore {
/*
* OBSOLETE MODIFICATION //SPECIAL HANDLING FOR MIRA CLASSIFIER PARAMETERS String[] paramA
* = line.split("\\s+");
- *
+ *
* if( paramA[0].equals("-classifierParams") ) { String classifierParam = ""; for(int p=1;
* p<=paramA.length-1; p++) classifierParam += paramA[p]+" ";
- *
+ *
* if(paramA.length>=2) { String[] tmpParamA = new String[2]; tmpParamA[0] = paramA[0];
* tmpParamA[1] = classifierParam; paramA = tmpParamA; } else {
* println("Malformed line in config file:"); println(origLine); System.exit(70); } }//END
@@ -2559,12 +2551,12 @@ public class MIRACore {
/*
* 1: -docSet bottom 8d 2: -docSet bottom 25% the bottom ceil(0.20*numDocs) documents 3: -docSet
* top 8d 4: -docSet top 25% the top ceil(0.20*numDocs) documents
- *
+ *
* 5: -docSet window 11d around 90percentile 11 docs centered around 80th percentile (complain
* if not enough docs; don't adjust) 6: -docSet window 11d around 40rank 11 docs centered around
* doc ranked 50 (complain if not enough docs; don't adjust)
- *
- *
+ *
+ *
* [0]: method (0-6) [1]: first (1-indexed) [2]: last (1-indexed) [3]: size [4]: center [5]:
* arg1 (-1 for method 0) [6]: arg2 (-1 for methods 0-4)
*/
@@ -2723,10 +2715,10 @@ public class MIRACore {
} else {
nextIndex = 1;
}
- int lineCount = countLines(prefix + nextIndex);
+ int lineCount = new ExistingUTF8EncodedTextFile(prefix + nextIndex).getNumberOfLines();
for (int r = 0; r < numFiles; ++r) {
- if (countLines(prefix + nextIndex) != lineCount) {
+ if (new ExistingUTF8EncodedTextFile(prefix + nextIndex).getNumberOfLines() != lineCount) {
throw new RuntimeException("Line count mismatch in " + (prefix + nextIndex) + ".");
}
InputStream inStream = new FileInputStream(new File(prefix + nextIndex));
@@ -2887,109 +2879,11 @@ public class MIRACore {
return str;
}
- private int countLines(String fileName) {
- int count = 0;
-
- try {
- BufferedReader inFile = new BufferedReader(new FileReader(fileName));
-
- String line;
- do {
- line = inFile.readLine();
- if (line != null)
- ++count;
- } while (line != null);
-
- inFile.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- return count;
- }
-
- private int countNonEmptyLines(String fileName) {
- int count = 0;
-
- try {
- BufferedReader inFile = new BufferedReader(new FileReader(fileName));
-
- String line;
- do {
- line = inFile.readLine();
- if (line != null && line.length() > 0)
- ++count;
- } while (line != null);
-
- inFile.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- return count;
- }
-
private String fullPath(String dir, String fileName) {
File dummyFile = new File(dir, fileName);
return dummyFile.getAbsolutePath();
}
- @SuppressWarnings("unused")
- private void cleanupMemory() {
- cleanupMemory(100, false);
- }
-
- @SuppressWarnings("unused")
- private void cleanupMemorySilently() {
- cleanupMemory(100, true);
- }
-
- @SuppressWarnings("static-access")
- private void cleanupMemory(int reps, boolean silent) {
- int bytesPerMB = 1024 * 1024;
-
- long totalMemBefore = myRuntime.totalMemory();
- long freeMemBefore = myRuntime.freeMemory();
- long usedMemBefore = totalMemBefore - freeMemBefore;
-
- long usedCurr = usedMemBefore;
- long usedPrev = usedCurr;
-
- // perform garbage collection repeatedly, until there is no decrease in
- // the amount of used memory
- for (int i = 1; i <= reps; ++i) {
- myRuntime.runFinalization();
- myRuntime.gc();
- (Thread.currentThread()).yield();
-
- usedPrev = usedCurr;
- usedCurr = myRuntime.totalMemory() - myRuntime.freeMemory();
-
- if (usedCurr == usedPrev)
- break;
- }
-
- if (!silent) {
- long totalMemAfter = myRuntime.totalMemory();
- long freeMemAfter = myRuntime.freeMemory();
- long usedMemAfter = totalMemAfter - freeMemAfter;
-
- println("GC: d_used = " + ((usedMemAfter - usedMemBefore) / bytesPerMB) + " MB "
- + "(d_tot = " + ((totalMemAfter - totalMemBefore) / bytesPerMB) + " MB).", 2);
- }
- }
-
- @SuppressWarnings("unused")
- private void printMemoryUsage() {
- int bytesPerMB = 1024 * 1024;
- long totalMem = myRuntime.totalMemory();
- long freeMem = myRuntime.freeMemory();
- long usedMem = totalMem - freeMem;
-
- println("Allocated memory: " + (totalMem / bytesPerMB) + " MB " + "(of which "
- + (usedMem / bytesPerMB) + " MB is being used).", 2);
- }
-
private void println(Object obj, int priority) {
if (priority <= verbosity)
println(obj);
@@ -3008,20 +2902,12 @@ public class MIRACore {
System.out.print(obj);
}
- @SuppressWarnings("unused")
- private void showProgress() {
- ++progress;
- if (progress % 100000 == 0)
- print(".", 2);
- }
-
private ArrayList<Double> randomLambda() {
ArrayList<Double> retLambda = new ArrayList<Double>(1 + numParams);
for (int c = 1; c <= numParams; ++c) {
if (isOptimizable[c]) {
double randVal = randGen.nextDouble(); // number in [0.0,1.0]
- ++generatedRands;
randVal = randVal * (maxRandValue[c] - minRandValue[c]); // number in [0.0,max-min]
randVal = minRandValue[c] + randVal; // number in [min,max]
retLambda.set(c, randVal);
@@ -3032,81 +2918,4 @@ public class MIRACore {
return retLambda;
}
-
- private double[] randomPerturbation(double[] origLambda, int i, double method, double param,
- double mult) {
- double sigma = 0.0;
- if (method == 1) {
- sigma = 1.0 / Math.pow(i, param);
- } else if (method == 2) {
- sigma = Math.exp(-param * i);
- } else if (method == 3) {
- sigma = Math.max(0.0, 1.0 - (i / param));
- }
-
- sigma = mult * sigma;
-
- double[] retLambda = new double[1 + numParams];
-
- for (int c = 1; c <= numParams; ++c) {
- if (isOptimizable[c]) {
- double randVal = 2 * randGen.nextDouble() - 1.0; // number in [-1.0,1.0]
- ++generatedRands;
- randVal = randVal * sigma; // number in [-sigma,sigma]
- randVal = randVal * origLambda[c]; // number in [-sigma*orig[c],sigma*orig[c]]
- randVal = randVal + origLambda[c]; // number in
- // [orig[c]-sigma*orig[c],orig[c]+sigma*orig[c]]
- // = [orig[c]*(1-sigma),orig[c]*(1+sigma)]
- retLambda[c] = randVal;
- } else {
- retLambda[c] = origLambda[c];
- }
- }
-
- return retLambda;
- }
-
- @SuppressWarnings("unused")
- private HashSet<Integer> indicesToDiscard(double[] slope, double[] offset) {
- // some lines can be eliminated: the ones that have a lower offset
- // than some other line with the same slope.
- // That is, for any k1 and k2:
- // if slope[k1] = slope[k2] and offset[k1] > offset[k2],
- // then k2 can be eliminated.
- // (This is actually important to do as it eliminates a bug.)
- // print("discarding: ",4);
-
- int numCandidates = slope.length;
- HashSet<Integer> discardedIndices = new HashSet<Integer>();
- HashMap<Double, Integer> indicesOfSlopes = new HashMap<Double, Integer>();
- // maps slope to index of best candidate that has that slope.
- // ("best" as in the one with the highest offset)
-
- for (int k1 = 0; k1 < numCandidates; ++k1) {
- double currSlope = slope[k1];
- if (!indicesOfSlopes.containsKey(currSlope)) {
- indicesOfSlopes.put(currSlope, k1);
- } else {
- int existingIndex = indicesOfSlopes.get(currSlope);
- if (offset[existingIndex] > offset[k1]) {
- discardedIndices.add(k1);
- // print(k1 + " ",4);
- } else if (offset[k1] > offset[existingIndex]) {
- indicesOfSlopes.put(currSlope, k1);
- discardedIndices.add(existingIndex);
- // print(existingIndex + " ",4);
- }
- }
- }
-
- // old way of doing it; takes quadratic time (vs. linear time above)
- /*
- * for (int k1 = 0; k1 < numCandidates; ++k1) { for (int k2 = 0; k2 < numCandidates; ++k2) { if
- * (k1 != k2 && slope[k1] == slope[k2] && offset[k1] > offset[k2]) { discardedIndices.add(k2);
- * // print(k2 + " ",4); } } }
- */
-
- // println("",4);
- return discardedIndices;
- } // indicesToDiscard(double[] slope, double[] offset)
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/mira/Optimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/mira/Optimizer.java b/src/main/java/org/apache/joshua/mira/Optimizer.java
index 6eaced4..f51a5b3 100755
--- a/src/main/java/org/apache/joshua/mira/Optimizer.java
+++ b/src/main/java/org/apache/joshua/mira/Optimizer.java
@@ -18,9 +18,9 @@
*/
package org.apache.joshua.mira;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -81,8 +81,6 @@ public class Optimizer {
String[] featInfo;
int thisBatchSize = 0;
int numBatch = 0;
- int numUpdate = 0;
- Iterator it;
Integer diffFeatId;
// update weights
@@ -98,7 +96,7 @@ public class Optimizer {
s = sents.get(sentCount);
// find out oracle and prediction
findOraPred(s, oraPredScore, oraPredFeat, finalLambda, featScale);
-
+
// the model scores here are already scaled in findOraPred
oraMetric = oraPredScore[0];
oraScore = oraPredScore[1];
@@ -106,7 +104,7 @@ public class Optimizer {
predScore = oraPredScore[3];
oraFeat = oraPredFeat[0];
predFeat = oraPredFeat[1];
-
+
// update the scale
if (needScale) { // otherwise featscale remains 1.0
sumMetricScore += java.lang.Math.abs(oraMetric + predMetric);
@@ -119,7 +117,7 @@ public class Optimizer {
vecOraFeat = oraFeat.split("\\s+");
vecPredFeat = predFeat.split("\\s+");
-
+
//accumulate difference feature vector
if ( b == 0 ) {
for (int i = 0; i < vecOraFeat.length; i++) {
@@ -185,8 +183,8 @@ public class Optimizer {
if (!runPercep) { // otherwise eta=1.0
featNorm = 0;
Collection<Double> allDiff = featDiff.values();
- for (it = allDiff.iterator(); it.hasNext();) {
- diff = (Double) it.next();
+ for (Iterator<Double> it = allDiff.iterator(); it.hasNext();) {
+ diff = it.next();
featNorm += diff * diff / ( thisBatchSize * thisBatchSize );
}
}
@@ -199,10 +197,10 @@ public class Optimizer {
}
avgEta += eta;
Set<Integer> diffFeatSet = featDiff.keySet();
- it = diffFeatSet.iterator();
+ Iterator<Integer> it = diffFeatSet.iterator();
if ( java.lang.Math.abs(eta) > 1e-20 ) {
while (it.hasNext()) {
- diffFeatId = (Integer) it.next();
+ diffFeatId = it.next();
finalLambda[diffFeatId] =
finalLambda[diffFeatId] + eta * featDiff.get(diffFeatId) / thisBatchSize;
}
@@ -304,7 +302,7 @@ public class Optimizer {
// find out the 1-best candidate for each sentence
// this depends on the training mode
maxModelScore = NegInf;
- for (Iterator it = candSet.iterator(); it.hasNext();) {
+ for (Iterator<String> it = candSet.iterator(); it.hasNext();) {
modelScore = 0.0;
candStr = it.next().toString();
feat_str = feat_hash[i].get(candStr).split("\\s+");
@@ -363,7 +361,7 @@ public class Optimizer {
worstPredScore = PosInf;
}
- for (Iterator it = candSet.iterator(); it.hasNext();) {
+ for (Iterator<String> it = candSet.iterator(); it.hasNext();) {
cand = it.next().toString();
candMetric = computeSentMetric(sentId, cand); // compute metric score
@@ -605,11 +603,11 @@ public class Optimizer {
}
}
}
-
+
public double getMetricScore() {
return finalMetricScore;
}
-
+
private Vector<String> output;
private double[] initialLambda;
private double[] finalLambda;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/pro/PROCore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/pro/PROCore.java b/src/main/java/org/apache/joshua/pro/PROCore.java
index ec23e0a..5dc3311 100755
--- a/src/main/java/org/apache/joshua/pro/PROCore.java
+++ b/src/main/java/org/apache/joshua/pro/PROCore.java
@@ -31,6 +31,7 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -49,7 +50,7 @@ import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.metrics.EvaluationMetric;
import org.apache.joshua.util.StreamGobbler;
-
+import org.apache.joshua.util.io.ExistingUTF8EncodedTextFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,19 +66,15 @@ public class PROCore {
private TreeSet<Integer>[] indicesOfInterest_all;
private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
- private final Runtime myRuntime = Runtime.getRuntime();
private final static double NegInf = (-1.0 / 0.0);
private final static double PosInf = (+1.0 / 0.0);
private final static double epsilon = 1.0 / 1000000;
- private int progress;
-
private int verbosity; // anything of priority <= verbosity will be printed
// (lower value for priority means more important)
private Random randGen;
- private int generatedRands;
private int numSentences;
// number of sentences in the dev set
@@ -256,28 +253,27 @@ public class PROCore {
this.joshuaConfiguration = joshuaConfiguration;
}
- public PROCore(String[] args, JoshuaConfiguration joshuaConfiguration) {
+ public PROCore(String[] args, JoshuaConfiguration joshuaConfiguration) throws FileNotFoundException, IOException {
this.joshuaConfiguration = joshuaConfiguration;
EvaluationMetric.set_knownMetrics();
processArgsArray(args);
initialize(0);
}
- public PROCore(String configFileName, JoshuaConfiguration joshuaConfiguration) {
+ public PROCore(String configFileName, JoshuaConfiguration joshuaConfiguration) throws FileNotFoundException, IOException {
this.joshuaConfiguration = joshuaConfiguration;
EvaluationMetric.set_knownMetrics();
processArgsArray(cfgFileToArgsArray(configFileName));
initialize(0);
}
- private void initialize(int randsToSkip) {
+ private void initialize(int randsToSkip) throws FileNotFoundException, IOException {
println("NegInf: " + NegInf + ", PosInf: " + PosInf + ", epsilon: " + epsilon, 4);
randGen = new Random(seed);
for (int r = 1; r <= randsToSkip; ++r) {
randGen.nextDouble();
}
- generatedRands = randsToSkip;
if (randsToSkip == 0) {
println("----------------------------------------------------", 1);
@@ -291,7 +287,7 @@ public class PROCore {
// COUNT THE TOTAL NUM OF SENTENCES TO BE DECODED, refFileName IS THE COMBINED REFERENCE FILE
// NAME(AUTO GENERATED)
- numSentences = countLines(refFileName) / refsPerSen;
+ numSentences = new ExistingUTF8EncodedTextFile(refFileName).getNumberOfLines() / refsPerSen;
// ??
processDocInfo();
@@ -304,7 +300,7 @@ public class PROCore {
set_docSubsetInfo(docSubsetInfo);
// count the number of initial features
- numParams = countNonEmptyLines(paramsFileName) - 1;
+ numParams = new ExistingUTF8EncodedTextFile(paramsFileName).getNumberOfNonEmptyLines() - 1;
numParamsOld = numParams;
// read parameter config file
@@ -910,7 +906,6 @@ public class PROCore {
for (String featurePair : featVal_str) {
String[] pair = featurePair.split("=");
String name = pair[0];
- Double value = Double.parseDouble(pair[1]);
int featId = Vocabulary.id(name);
// need to identify newly fired feats here
if (featId > numParams) {
@@ -1573,8 +1568,6 @@ public class PROCore {
BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
PrintWriter outFile = new PrintWriter(cfgFileName);
- BufferedReader inFeatDefFile = null;
- PrintWriter outFeatDefFile = null;
int origFeatNum = 0; // feat num in the template file
String line = inFile.readLine();
@@ -1776,7 +1769,7 @@ public class PROCore {
// belongs to,
// and its order in that document. (can also use '-' instead of '_')
- int docInfoSize = countNonEmptyLines(docInfoFileName);
+ int docInfoSize = new ExistingUTF8EncodedTextFile(docInfoFileName).getNumberOfNonEmptyLines();
if (docInfoSize < numSentences) { // format #1 or #2
numDocuments = docInfoSize;
@@ -1969,9 +1962,7 @@ public class PROCore {
Vector<String> argsVector = new Vector<String>();
- BufferedReader inFile = null;
- try {
- inFile = new BufferedReader(new FileReader(fileName));
+ try (BufferedReader inFile = new BufferedReader(new FileReader(fileName));) {
String line, origLine;
do {
line = inFile.readLine();
@@ -2044,8 +2035,6 @@ public class PROCore {
}
} while (line != null);
-
- inFile.close();
} catch (FileNotFoundException e) {
println("PRO configuration file " + fileName + " was not found!");
throw new RuntimeException(e);
@@ -2626,9 +2615,7 @@ public class PROCore {
outFileName = prefix + ".all";
}
- try {
- PrintWriter outFile = new PrintWriter(outFileName);
-
+ try (PrintWriter outFile = new PrintWriter(outFileName);) {
BufferedReader[] inFile = new BufferedReader[numFiles];
int nextIndex;
@@ -2638,14 +2625,14 @@ public class PROCore {
} else {
nextIndex = 1;
}
- int lineCount = countLines(prefix + nextIndex);
+ int lineCount = new ExistingUTF8EncodedTextFile(prefix + nextIndex).getNumberOfLines();
for (int r = 0; r < numFiles; ++r) {
- if (countLines(prefix + nextIndex) != lineCount) {
+ if (new ExistingUTF8EncodedTextFile(prefix + nextIndex).getNumberOfLines() != lineCount) {
throw new RuntimeException("Line count mismatch in " + (prefix + nextIndex) + ".");
}
InputStream inStream = new FileInputStream(new File(prefix + nextIndex));
- inFile[r] = new BufferedReader(new InputStreamReader(inStream, "utf8"));
+ inFile[r] = new BufferedReader(new InputStreamReader(inStream, StandardCharsets.UTF_8));
++nextIndex;
}
@@ -2658,8 +2645,6 @@ public class PROCore {
}
}
- outFile.close();
-
for (int r = 0; r < numFiles; ++r) {
inFile[r].close();
}
@@ -2802,109 +2787,11 @@ public class PROCore {
return str;
}
- private int countLines(String fileName) {
- int count = 0;
-
- try {
- BufferedReader inFile = new BufferedReader(new FileReader(fileName));
-
- String line;
- do {
- line = inFile.readLine();
- if (line != null)
- ++count;
- } while (line != null);
-
- inFile.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- return count;
- }
-
- private int countNonEmptyLines(String fileName) {
- int count = 0;
-
- try {
- BufferedReader inFile = new BufferedReader(new FileReader(fileName));
-
- String line;
- do {
- line = inFile.readLine();
- if (line != null && line.length() > 0)
- ++count;
- } while (line != null);
-
- inFile.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- return count;
- }
-
private String fullPath(String dir, String fileName) {
File dummyFile = new File(dir, fileName);
return dummyFile.getAbsolutePath();
}
- @SuppressWarnings("unused")
- private void cleanupMemory() {
- cleanupMemory(100, false);
- }
-
- @SuppressWarnings("unused")
- private void cleanupMemorySilently() {
- cleanupMemory(100, true);
- }
-
- @SuppressWarnings("static-access")
- private void cleanupMemory(int reps, boolean silent) {
- int bytesPerMB = 1024 * 1024;
-
- long totalMemBefore = myRuntime.totalMemory();
- long freeMemBefore = myRuntime.freeMemory();
- long usedMemBefore = totalMemBefore - freeMemBefore;
-
- long usedCurr = usedMemBefore;
- long usedPrev = usedCurr;
-
- // perform garbage collection repeatedly, until there is no decrease in
- // the amount of used memory
- for (int i = 1; i <= reps; ++i) {
- myRuntime.runFinalization();
- myRuntime.gc();
- (Thread.currentThread()).yield();
-
- usedPrev = usedCurr;
- usedCurr = myRuntime.totalMemory() - myRuntime.freeMemory();
-
- if (usedCurr == usedPrev)
- break;
- }
-
- if (!silent) {
- long totalMemAfter = myRuntime.totalMemory();
- long freeMemAfter = myRuntime.freeMemory();
- long usedMemAfter = totalMemAfter - freeMemAfter;
-
- println("GC: d_used = " + ((usedMemAfter - usedMemBefore) / bytesPerMB) + " MB "
- + "(d_tot = " + ((totalMemAfter - totalMemBefore) / bytesPerMB) + " MB).", 2);
- }
- }
-
- @SuppressWarnings("unused")
- private void printMemoryUsage() {
- int bytesPerMB = 1024 * 1024;
- long totalMem = myRuntime.totalMemory();
- long freeMem = myRuntime.freeMemory();
- long usedMem = totalMem - freeMem;
-
- println("Allocated memory: " + (totalMem / bytesPerMB) + " MB " + "(of which "
- + (usedMem / bytesPerMB) + " MB is being used).", 2);
- }
-
private void println(Object obj, int priority) {
if (priority <= verbosity)
println(obj);
@@ -2923,20 +2810,12 @@ public class PROCore {
System.out.print(obj);
}
- @SuppressWarnings("unused")
- private void showProgress() {
- ++progress;
- if (progress % 100000 == 0)
- print(".", 2);
- }
-
private ArrayList<Double> randomLambda() {
ArrayList<Double> retLambda = new ArrayList<Double>(1 + numParams);
for (int c = 1; c <= numParams; ++c) {
if (isOptimizable[c]) {
double randVal = randGen.nextDouble(); // number in [0.0,1.0]
- ++generatedRands;
randVal = randVal * (maxRandValue[c] - minRandValue[c]); // number in [0.0,max-min]
randVal = minRandValue[c] + randVal; // number in [min,max]
retLambda.set(c, randVal);
@@ -2947,81 +2826,4 @@ public class PROCore {
return retLambda;
}
-
- private double[] randomPerturbation(double[] origLambda, int i, double method, double param,
- double mult) {
- double sigma = 0.0;
- if (method == 1) {
- sigma = 1.0 / Math.pow(i, param);
- } else if (method == 2) {
- sigma = Math.exp(-param * i);
- } else if (method == 3) {
- sigma = Math.max(0.0, 1.0 - (i / param));
- }
-
- sigma = mult * sigma;
-
- double[] retLambda = new double[1 + numParams];
-
- for (int c = 1; c <= numParams; ++c) {
- if (isOptimizable[c]) {
- double randVal = 2 * randGen.nextDouble() - 1.0; // number in [-1.0,1.0]
- ++generatedRands;
- randVal = randVal * sigma; // number in [-sigma,sigma]
- randVal = randVal * origLambda[c]; // number in [-sigma*orig[c],sigma*orig[c]]
- randVal = randVal + origLambda[c]; // number in
- // [orig[c]-sigma*orig[c],orig[c]+sigma*orig[c]]
- // = [orig[c]*(1-sigma),orig[c]*(1+sigma)]
- retLambda[c] = randVal;
- } else {
- retLambda[c] = origLambda[c];
- }
- }
-
- return retLambda;
- }
-
- @SuppressWarnings("unused")
- private HashSet<Integer> indicesToDiscard(double[] slope, double[] offset) {
- // some lines can be eliminated: the ones that have a lower offset
- // than some other line with the same slope.
- // That is, for any k1 and k2:
- // if slope[k1] = slope[k2] and offset[k1] > offset[k2],
- // then k2 can be eliminated.
- // (This is actually important to do as it eliminates a bug.)
- // print("discarding: ",4);
-
- int numCandidates = slope.length;
- HashSet<Integer> discardedIndices = new HashSet<Integer>();
- HashMap<Double, Integer> indicesOfSlopes = new HashMap<Double, Integer>();
- // maps slope to index of best candidate that has that slope.
- // ("best" as in the one with the highest offset)
-
- for (int k1 = 0; k1 < numCandidates; ++k1) {
- double currSlope = slope[k1];
- if (!indicesOfSlopes.containsKey(currSlope)) {
- indicesOfSlopes.put(currSlope, k1);
- } else {
- int existingIndex = indicesOfSlopes.get(currSlope);
- if (offset[existingIndex] > offset[k1]) {
- discardedIndices.add(k1);
- // print(k1 + " ",4);
- } else if (offset[k1] > offset[existingIndex]) {
- indicesOfSlopes.put(currSlope, k1);
- discardedIndices.add(existingIndex);
- // print(existingIndex + " ",4);
- }
- }
- }
-
- // old way of doing it; takes quadratic time (vs. linear time above)
- /*
- * for (int k1 = 0; k1 < numCandidates; ++k1) { for (int k2 = 0; k2 < numCandidates; ++k2) { if
- * (k1 != k2 && slope[k1] == slope[k2] && offset[k1] > offset[k2]) { discardedIndices.add(k2);
- * // print(k2 + " ",4); } } }
- */
-
- // println("",4);
- return discardedIndices;
- } // indicesToDiscard(double[] slope, double[] offset)
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/tools/GrammarPacker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/tools/GrammarPacker.java b/src/main/java/org/apache/joshua/tools/GrammarPacker.java
index b39b775..e416d03 100644
--- a/src/main/java/org/apache/joshua/tools/GrammarPacker.java
+++ b/src/main/java/org/apache/joshua/tools/GrammarPacker.java
@@ -54,13 +54,13 @@ public class GrammarPacker {
/**
* The packed grammar version number. Increment this any time you add new features, and update
* the documentation.
- *
+ *
* Version history:
- *
+ *
* - 3 (May 2016). This was the first version that was marked. It removed the special phrase-
* table packing that packed phrases without the [X,1] on the source and target sides, which
* then required special handling in the decoder to use for phrase-based decoding.
- *
+ *
* - 4 (August 2016). Phrase-based decoding rewritten to represent phrases without a builtin
* nonterminal. Instead, cost-less glue rules are used in phrase-based decoding. This eliminates
* the need for special handling of phrase grammars (except for having to add a LHS), and lets
@@ -68,7 +68,7 @@ public class GrammarPacker {
*
*/
public static final int VERSION = 4;
-
+
// Size limit for slice in bytes.
private static int DATA_SIZE_LIMIT = (int) (Integer.MAX_VALUE * 0.8);
// Estimated average number of feature entries for one rule.
@@ -145,30 +145,30 @@ public class GrammarPacker {
}
private void readConfig(String config_filename) throws IOException {
- LineReader reader = new LineReader(config_filename);
- while (reader.hasNext()) {
- // Clean up line, chop comments off and skip if the result is empty.
- String line = reader.next().trim();
- if (line.indexOf('#') != -1)
- line = line.substring(0, line.indexOf('#'));
- if (line.isEmpty())
- continue;
- String[] fields = line.split("[\\s]+");
-
- if (fields.length < 2) {
- throw new RuntimeException("Incomplete line in config.");
- }
- if ("slice_size".equals(fields[0])) {
- // Number of records to concurrently load into memory for sorting.
- approximateMaximumSliceSize = Integer.parseInt(fields[1]);
+ try(LineReader reader = new LineReader(config_filename);) {
+ while (reader.hasNext()) {
+ // Clean up line, chop comments off and skip if the result is empty.
+ String line = reader.next().trim();
+ if (line.indexOf('#') != -1)
+ line = line.substring(0, line.indexOf('#'));
+ if (line.isEmpty())
+ continue;
+ String[] fields = line.split("[\\s]+");
+
+ if (fields.length < 2) {
+ throw new RuntimeException("Incomplete line in config.");
+ }
+ if ("slice_size".equals(fields[0])) {
+ // Number of records to concurrently load into memory for sorting.
+ approximateMaximumSliceSize = Integer.parseInt(fields[1]);
+ }
}
}
- reader.close();
}
/**
* Executes the packing.
- *
+ *
* @throws IOException if there is an error reading the grammar
*/
public void pack() throws IOException {
@@ -226,23 +226,24 @@ public class GrammarPacker {
/**
* Returns a reader that turns whatever file format is found into Hiero grammar rules.
- *
+ *
* @param grammarFile
* @return
* @throws IOException
*/
private HieroFormatReader getGrammarReader() throws IOException {
- LineReader reader = new LineReader(grammar);
- String line = reader.next();
- if (line.startsWith("[")) {
- return new HieroFormatReader(grammar);
- } else {
- return new MosesFormatReader(grammar);
+ try (LineReader reader = new LineReader(grammar);) {
+ String line = reader.next();
+ if (line.startsWith("[")) {
+ return new HieroFormatReader(grammar);
+ } else {
+ return new MosesFormatReader(grammar);
+ }
}
}
/**
- * This first pass over the grammar
+ * This first pass over the grammar
* @param reader
*/
private void explore(HieroFormatReader reader) {
@@ -258,9 +259,9 @@ public class GrammarPacker {
/* Add symbols to vocabulary.
* NOTE: In case of nonterminals, we add both stripped versions ("[X]")
* and "[X,1]" to the vocabulary.
- *
+ *
* TODO: MJP May 2016: Is it necessary to add [X,1]? This is currently being done in
- * {@link HieroFormatReader}, which is called by {@link MosesFormatReader}.
+ * {@link HieroFormatReader}, which is called by {@link MosesFormatReader}.
*/
// Add feature names to vocabulary and pass the value through the
@@ -438,7 +439,7 @@ public class GrammarPacker {
* written simultaneously. The source structure is written into a downward-pointing trie and
* stores the rule's lhs as well as links to the target and feature stream. The feature stream is
* prompted to write out a block
- *
+ *
* @param source_trie
* @param target_trie
* @param feature_buffer
@@ -580,9 +581,9 @@ public class GrammarPacker {
/**
* Integer-labeled, doubly-linked trie with some provisions for packing.
- *
+ *
* @author Juri Ganitkevitch
- *
+ *
* @param <D> The trie's value type.
*/
class PackingTrie<D extends PackingTrieValue> {
@@ -632,10 +633,10 @@ public class GrammarPacker {
* points to children) from upwards pointing (children point to parent) tries, as well as
* skeletal (no data, just the labeled links) and non-skeletal (nodes have a data block)
* packing.
- *
+ *
* @param downwards Are we packing into a downwards-pointing trie?
* @param skeletal Are we packing into a skeletal trie?
- *
+ *
* @return Number of bytes the trie node would occupy.
*/
int size(boolean downwards, boolean skeletal) {
@@ -684,6 +685,7 @@ public class GrammarPacker {
this.target = target;
}
+ @Override
public int size() {
return 3;
}
@@ -696,6 +698,7 @@ public class GrammarPacker {
this.parent = parent;
}
+ @Override
public int size() {
return 0;
}
@@ -753,7 +756,7 @@ public class GrammarPacker {
/**
* Enqueue a data block for later writing.
- *
+ *
* @param block_index The index of the data block to add to writing queue.
* @return The to-be-written block's output index.
*/
@@ -765,7 +768,7 @@ public class GrammarPacker {
/**
* Performs the actual writing to disk in the order specified by calls to write() since the last
* call to initialize().
- *
+ *
* @param out
* @throws IOException
*/
@@ -828,10 +831,11 @@ public class GrammarPacker {
/**
* Add a block of features to the buffer.
- *
+ *
* @param features TreeMap with the features for one rule.
* @return The index of the resulting data block.
*/
+ @Override
int add(TreeMap<Integer, Float> features) {
int data_position = buffer.position();
@@ -871,10 +875,11 @@ public class GrammarPacker {
/**
* Add a rule alignments to the buffer.
- *
+ *
* @param alignments a byte array with the alignment points for one rule.
* @return The index of the resulting data block.
*/
+ @Override
int add(byte[] alignments) {
int data_position = buffer.position();
int size_estimate = alignments.length + 1;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/tools/LabelPhrases.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/tools/LabelPhrases.java b/src/main/java/org/apache/joshua/tools/LabelPhrases.java
index 2fd2b3f..8f15b0e 100644
--- a/src/main/java/org/apache/joshua/tools/LabelPhrases.java
+++ b/src/main/java/org/apache/joshua/tools/LabelPhrases.java
@@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory;
/**
* Finds labeling for a set of phrases.
- *
+ *
* @author Juri Ganitkevitch
*/
public class LabelPhrases {
@@ -37,7 +37,7 @@ public class LabelPhrases {
/**
* Main method.
- *
+ *
* @param args names of the two grammars to be compared
* @throws IOException if there is an error reading the input grammars
*/
@@ -60,52 +60,52 @@ public class LabelPhrases {
System.exit(-1);
}
- LineReader phrase_reader = new LineReader(phrase_file_name);
-
- while (phrase_reader.ready()) {
- String line = phrase_reader.readLine();
-
- String[] fields = line.split("\\t");
- if (fields.length != 3 || fields[2].equals("()")) {
- System.err.println("[FAIL] Empty parse in line:\t" + line);
- continue;
- }
-
- String[] phrase_strings = fields[0].split("\\s");
- int[] phrase_ids = new int[phrase_strings.length];
- for (int i = 0; i < phrase_strings.length; i++)
- phrase_ids[i] = Vocabulary.id(phrase_strings[i]);
+ try (LineReader phrase_reader = new LineReader(phrase_file_name);) {
+ while (phrase_reader.ready()) {
+ String line = phrase_reader.readLine();
- ArraySyntaxTree syntax = new ArraySyntaxTree(fields[2]);
- int[] sentence_ids = syntax.getTerminals();
+ String[] fields = line.split("\\t");
+ if (fields.length != 3 || fields[2].equals("()")) {
+ System.err.println("[FAIL] Empty parse in line:\t" + line);
+ continue;
+ }
- int match_start = -1;
- int match_end = -1;
- for (int i = 0; i < sentence_ids.length; i++) {
- if (phrase_ids[0] == sentence_ids[i]) {
- match_start = i;
- int j = 0;
- while (j < phrase_ids.length && phrase_ids[j] == sentence_ids[i + j]) {
- j++;
- }
- if (j == phrase_ids.length) {
- match_end = i + j;
- break;
+ String[] phrase_strings = fields[0].split("\\s");
+ int[] phrase_ids = new int[phrase_strings.length];
+ for (int i = 0; i < phrase_strings.length; i++)
+ phrase_ids[i] = Vocabulary.id(phrase_strings[i]);
+
+ ArraySyntaxTree syntax = new ArraySyntaxTree(fields[2]);
+ int[] sentence_ids = syntax.getTerminals();
+
+ int match_start = -1;
+ int match_end = -1;
+ for (int i = 0; i < sentence_ids.length; i++) {
+ if (phrase_ids[0] == sentence_ids[i]) {
+ match_start = i;
+ int j = 0;
+ while (j < phrase_ids.length && phrase_ids[j] == sentence_ids[i + j]) {
+ j++;
+ }
+ if (j == phrase_ids.length) {
+ match_end = i + j;
+ break;
+ }
}
}
- }
- int label = syntax.getOneConstituent(match_start, match_end);
- if (label == 0) label = syntax.getOneSingleConcatenation(match_start, match_end);
- if (label == 0) label = syntax.getOneRightSideCCG(match_start, match_end);
- if (label == 0) label = syntax.getOneLeftSideCCG(match_start, match_end);
- if (label == 0) label = syntax.getOneDoubleConcatenation(match_start, match_end);
- if (label == 0) {
- System.err.println("[FAIL] No label found in line:\t" + line);
- continue;
- }
+ int label = syntax.getOneConstituent(match_start, match_end);
+ if (label == 0) label = syntax.getOneSingleConcatenation(match_start, match_end);
+ if (label == 0) label = syntax.getOneRightSideCCG(match_start, match_end);
+ if (label == 0) label = syntax.getOneLeftSideCCG(match_start, match_end);
+ if (label == 0) label = syntax.getOneDoubleConcatenation(match_start, match_end);
+ if (label == 0) {
+ System.err.println("[FAIL] No label found in line:\t" + line);
+ continue;
+ }
- System.out.println(Vocabulary.word(label) + "\t" + line);
+ System.out.println(Vocabulary.word(label) + "\t" + line);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/tools/TestSetFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/tools/TestSetFilter.java b/src/main/java/org/apache/joshua/tools/TestSetFilter.java
index ecb2e6e..f73f02d 100644
--- a/src/main/java/org/apache/joshua/tools/TestSetFilter.java
+++ b/src/main/java/org/apache/joshua/tools/TestSetFilter.java
@@ -57,7 +57,7 @@ public class TestSetFilter {
acceptedLastSourceSide = false;
lastSourceSide = null;
}
-
+
public String getFilterName() {
if (filter != null)
if (filter instanceof FastFilter)
@@ -109,7 +109,7 @@ public class TestSetFilter {
}
/**
- * Top-level filter, responsible for calling the fast or exact version. Takes the source side
+ * Top-level filter, responsible for calling the fast or exact version. Takes the source side
* of a rule and determines whether there is any sentence in the test set that can match it.
* @param sourceSide an input source sentence
* @return true if is any sentence in the test set can match the source input
@@ -124,11 +124,11 @@ public class TestSetFilter {
return acceptedLastSourceSide;
}
-
+
/**
* Determines whether a rule is an abstract rule. An abstract rule is one that has no terminals on
* its source side.
- *
+ *
* If the rule is abstract, the rule's arity is returned. Otherwise, 0 is returned.
*/
private boolean isAbstract(String source) {
@@ -144,7 +144,7 @@ public class TestSetFilter {
private interface Filter {
/* Tell the filter about a sentence in the test set being filtered to */
public void addSentence(String sentence);
-
+
/* Returns true if the filter permits the specified source side */
public boolean permits(String sourceSide);
}
@@ -155,7 +155,7 @@ public class TestSetFilter {
public FastFilter() {
ngrams = new HashSet<String>();
}
-
+
@Override
public boolean permits(String source) {
for (String chunk : source.split(NT_REGEX)) {
@@ -194,7 +194,7 @@ public class TestSetFilter {
public LooseFilter() {
testSentences = new ArrayList<String>();
}
-
+
@Override
public void addSentence(String source) {
testSentences.add(source);
@@ -227,13 +227,13 @@ public class TestSetFilter {
private FastFilter fastFilter = null;
private Map<String, Set<Integer>> sentencesByWord;
List<String> testSentences = null;
-
+
public ExactFilter() {
fastFilter = new FastFilter();
sentencesByWord = new HashMap<String, Set<Integer>>();
testSentences = new ArrayList<String>();
}
-
+
@Override
public void addSentence(String source) {
fastFilter.addSentence(source);
@@ -243,13 +243,13 @@ public class TestSetFilter {
/**
* Always permit abstract rules. Otherwise, query the fast filter, and if that passes, apply
- *
+ *
*/
@Override
public boolean permits(String sourceSide) {
if (isAbstract(sourceSide))
return true;
-
+
if (fastFilter.permits(sourceSide)) {
Pattern pattern = getPattern(sourceSide);
for (int i : getSentencesForRule(sourceSide)) {
@@ -257,10 +257,10 @@ public class TestSetFilter {
return true;
}
}
- }
+ }
return false;
}
-
+
protected Pattern getPattern(String source) {
String pattern = Pattern.quote(source);
pattern = pattern.replaceAll(NT_REGEX, "\\\\E.+\\\\Q");
@@ -268,7 +268,7 @@ public class TestSetFilter {
pattern = "(?:^|\\s)" + pattern + "(?:$|\\s)";
return Pattern.compile(pattern);
}
-
+
/*
* Map words to all the sentences they appear in.
*/
@@ -280,7 +280,7 @@ public class TestSetFilter {
sentencesByWord.get(t).add(index);
}
}
-
+
private Set<Integer> getSentencesForRule(String source) {
Set<Integer> sentences = null;
for (String token : source.split("\\s+")) {
@@ -293,7 +293,7 @@ public class TestSetFilter {
}
}
}
-
+
return sentences;
}
}
@@ -311,7 +311,7 @@ public class TestSetFilter {
System.err.println(" -n max n-gram to compare to (default 12)");
return;
}
-
+
String grammarFile = null;
TestSetFilter filter = new TestSetFilter();
@@ -350,34 +350,35 @@ public class TestSetFilter {
System.err.println(String.format("Filtering rules with the %s filter...", filter.getFilterName()));
// System.err.println("Using at max " + filter.RULE_LENGTH + " n-grams...");
}
- LineReader reader = (grammarFile != null)
+ try(LineReader reader = (grammarFile != null)
? new LineReader(grammarFile, filter.verbose)
- : new LineReader(System.in);
- for (String rule: reader) {
- rulesIn++;
-
- String[] parts = P_DELIM.split(rule);
- if (parts.length >= 4) {
- // the source is the second field for thrax grammars, first field for phrasal ones
- String source = rule.startsWith("[") ? parts[1].trim() : parts[0].trim();
- if (filter.inTestSet(source)) {
- System.out.println(rule);
- if (filter.parallel)
+ : new LineReader(System.in);) {
+ for (String rule: reader) {
+ rulesIn++;
+
+ String[] parts = P_DELIM.split(rule);
+ if (parts.length >= 4) {
+ // the source is the second field for thrax grammars, first field for phrasal ones
+ String source = rule.startsWith("[") ? parts[1].trim() : parts[0].trim();
+ if (filter.inTestSet(source)) {
+ System.out.println(rule);
+ if (filter.parallel)
+ System.out.flush();
+ rulesOut++;
+ } else if (filter.parallel) {
+ System.out.println("");
System.out.flush();
- rulesOut++;
- } else if (filter.parallel) {
- System.out.println("");
- System.out.flush();
+ }
}
}
- }
- if (filter.verbose) {
- System.err.println("[INFO] Total rules read: " + rulesIn);
- System.err.println("[INFO] Rules kept: " + rulesOut);
- System.err.println("[INFO] Rules dropped: " + (rulesIn - rulesOut));
- System.err.println("[INFO] cached queries: " + filter.cached);
- }
+ if (filter.verbose) {
+ System.err.println("[INFO] Total rules read: " + rulesIn);
+ System.err.println("[INFO] Rules kept: " + rulesOut);
+ System.err.println("[INFO] Rules dropped: " + (rulesIn - rulesOut));
+ System.err.println("[INFO] cached queries: " + filter.cached);
+ }
- return;
+ return;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/840eb4ce/src/main/java/org/apache/joshua/util/BotMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/util/BotMap.java b/src/main/java/org/apache/joshua/util/BotMap.java
deleted file mode 100644
index 1cc82b5..0000000
--- a/src/main/java/org/apache/joshua/util/BotMap.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.util;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Gets a special map that maps any key to the a particular value.
- *
- * @author Lane Schwartz
- * @see "Lopez (2008), footnote 9 on p73"
- */
-public class BotMap<K, V> implements Map<K, V> {
-
- /** Special value, which this map will return for every key. */
- private final V value;
-
- /**
- * Constructs a special map that maps any key to the a particular value.
- *
- * @param value Special value, which this map will return for every key.
- */
- public BotMap(V value) {
- this.value = value;
- }
-
- public void clear() {
- throw new UnsupportedOperationException();
- }
-
- public boolean containsKey(Object key) {
- return true;
- }
-
- public boolean containsValue(Object value) {
- return this.value == value;
- }
-
- public Set<Map.Entry<K, V>> entrySet() {
- throw new UnsupportedOperationException();
- }
-
- public V get(Object key) {
- return value;
- }
-
- public boolean isEmpty() {
- return false;
- }
-
- public Set<K> keySet() {
- throw new UnsupportedOperationException();
- }
-
- public V put(K key, V value) {
- throw new UnsupportedOperationException();
- }
-
- public void putAll(Map<? extends K, ? extends V> t) {
- throw new UnsupportedOperationException();
- }
-
- public V remove(Object key) {
- throw new UnsupportedOperationException();
- }
-
- public int size() {
- throw new UnsupportedOperationException();
- }
-
- public Collection<V> values() {
- return Collections.singleton(value);
- }
-
-}