You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/03 21:01:34 UTC
[03/14] incubator-joshua git commit: Large refactor of the
Translation output interface
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/f2f82c38/src/joshua/oracle/OracleExtractionHG.java
----------------------------------------------------------------------
diff --git a/src/joshua/oracle/OracleExtractionHG.java b/src/joshua/oracle/OracleExtractionHG.java
deleted file mode 100644
index 8a688e3..0000000
--- a/src/joshua/oracle/OracleExtractionHG.java
+++ /dev/null
@@ -1,794 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.oracle;
-
-import static joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
-import static joshua.util.FormatUtils.removeSentenceMarkers;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import joshua.corpus.Vocabulary;
-import joshua.decoder.JoshuaConfiguration;
-import joshua.decoder.Support;
-import joshua.decoder.Decoder;
-import joshua.decoder.hypergraph.HGNode;
-import joshua.decoder.hypergraph.HyperEdge;
-import joshua.decoder.hypergraph.HyperGraph;
-import joshua.decoder.hypergraph.KBestExtractor;
-import joshua.util.FileUtility;
-import joshua.util.FormatUtils;
-import joshua.util.io.LineReader;
-
-/**
- * approximated BLEU (1) do not consider clipping effect (2) in the dynamic programming, do not
- * maintain different states for different hyp length (3) brief penalty is calculated based on the
- * avg ref length (4) using sentence-level BLEU, instead of doc-level BLEU
- *
- * @author Zhifei Li, <zh...@gmail.com> (Johns Hopkins University)
- */
-public class OracleExtractionHG extends SplitHg {
- static String BACKOFF_LEFT_LM_STATE_SYM = "<lzfbo>";
- public int BACKOFF_LEFT_LM_STATE_SYM_ID;// used for equivelant state
-
- static String NULL_LEFT_LM_STATE_SYM = "<lzflnull>";
- public int NULL_LEFT_LM_STATE_SYM_ID;// used for equivelant state
-
- static String NULL_RIGHT_LM_STATE_SYM = "<lzfrnull>";
- public int NULL_RIGHT_LM_STATE_SYM_ID;// used for equivelant state
-
- // int[] ref_sentence;//reference string (not tree)
- protected int src_sent_len = 0;
- protected int ref_sent_len = 0;
- protected int g_lm_order = 4; // only used for decide whether to get the LM state by this class or
- // not in compute_state
- static protected boolean do_local_ngram_clip = false;
- static protected boolean maitain_length_state = false;
- static protected int g_bleu_order = 4;
-
- static boolean using_left_equiv_state = true;
- static boolean using_right_equiv_state = true;
-
- // TODO Add generics to hash tables in this class
- HashMap<String, Boolean> tbl_suffix = new HashMap<String, Boolean>();
- HashMap<String, Boolean> tbl_prefix = new HashMap<String, Boolean>();
- static PrefixGrammar grammar_prefix = new PrefixGrammar();// TODO
- static PrefixGrammar grammar_suffix = new PrefixGrammar();// TODO
-
- // key: item; value: best_deduction, best_bleu, best_len, # of n-gram match where n is in [1,4]
- protected HashMap<String, Integer> tbl_ref_ngrams = new HashMap<String, Integer>();
-
- static boolean always_maintain_seperate_lm_state = true; // if true: the virtual item maintain its
- // own lm state regardless whether
- // lm_order>=g_bleu_order
-
- int lm_feat_id = 0; // the baseline LM feature id
-
- /**
- * Constructs a new object capable of extracting a tree from a hypergraph that most closely
- * matches a provided oracle sentence.
- * <p>
- * It seems that the symbol table here should only need to represent monolingual terminals, plus
- * nonterminals.
- *
- * @param lm_feat_id_
- */
- public OracleExtractionHG(int lm_feat_id_) {
- this.lm_feat_id = lm_feat_id_;
- this.BACKOFF_LEFT_LM_STATE_SYM_ID = Vocabulary.id(BACKOFF_LEFT_LM_STATE_SYM);
- this.NULL_LEFT_LM_STATE_SYM_ID = Vocabulary.id(NULL_RIGHT_LM_STATE_SYM);
- this.NULL_RIGHT_LM_STATE_SYM_ID = Vocabulary.id(NULL_RIGHT_LM_STATE_SYM);
- }
-
- /*
- * for 919 sent, time_on_reading: 148797 time_on_orc_extract: 580286
- */
- @SuppressWarnings({ "unused" })
- public static void main(String[] args) throws IOException {
- JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
- /*
- * String f_hypergraphs="C:\\Users\\zli\\Documents\\mt03.src.txt.ss.nbest.hg.items"; String
- * f_rule_tbl="C:\\Users\\zli\\Documents\\mt03.src.txt.ss.nbest.hg.rules"; String
- * f_ref_files="C:\\Users\\zli\\Documents\\mt03.ref.txt.1"; String f_orc_out
- * ="C:\\Users\\zli\\Documents\\mt03.orc.txt";
- */
- if (6 != args.length) {
- System.out
- .println("Usage: java Decoder f_hypergraphs f_rule_tbl f_ref_files f_orc_out lm_order orc_extract_nbest");
- System.out.println("num of args is " + args.length);
- for (int i = 0; i < args.length; i++) {
- System.out.println("arg is: " + args[i]);
- }
- System.exit(1);
- }
- // String f_hypergraphs = args[0].trim();
- // String f_rule_tbl = args[1].trim();
- String f_ref_files = args[2].trim();
- String f_orc_out = args[3].trim();
- int lm_order = Integer.parseInt(args[4].trim());
- boolean orc_extract_nbest = Boolean.valueOf(args[5].trim()); // oracle extraction from nbest or
- // hg
-
- // ??????????????????????????????????????
- int baseline_lm_feat_id = 0;
- // ??????????????????????????????????????
-
- KBestExtractor kbest_extractor = null;
- int topN = 300;// TODO
- joshuaConfiguration.use_unique_nbest = true;
- joshuaConfiguration.include_align_index = false;
- boolean do_ngram_clip_nbest = true; // TODO
- if (orc_extract_nbest) {
- System.out.println("oracle extraction from nbest list");
-
- kbest_extractor = new KBestExtractor(null, null, Decoder.weights, false, joshuaConfiguration);
- }
-
- BufferedWriter orc_out = FileUtility.getWriteFileStream(f_orc_out);
-
- long start_time0 = System.currentTimeMillis();
- long time_on_reading = 0;
- long time_on_orc_extract = 0;
- // DiskHyperGraph dhg_read = new DiskHyperGraph(baseline_lm_feat_id, true, null);
-
- // dhg_read.initRead(f_hypergraphs, f_rule_tbl, null);
-
- OracleExtractionHG orc_extractor = new OracleExtractionHG(baseline_lm_feat_id);
- long start_time = System.currentTimeMillis();
- int sent_id = 0;
- for (String ref_sent: new LineReader(f_ref_files)) {
- System.out.println("############Process sentence " + sent_id);
- start_time = System.currentTimeMillis();
- sent_id++;
- // if(sent_id>10)break;
-
- // HyperGraph hg = dhg_read.readHyperGraph();
- HyperGraph hg = null;
- if (hg == null)
- continue;
-
- // System.out.println("read disk hyp: " + (System.currentTimeMillis()-start_time));
- time_on_reading += System.currentTimeMillis() - start_time;
- start_time = System.currentTimeMillis();
-
- String orc_sent = null;
- double orc_bleu = 0;
- if (orc_extract_nbest) {
- Object[] res = orc_extractor.oracle_extract_nbest(kbest_extractor, hg, topN,
- do_ngram_clip_nbest, ref_sent);
- orc_sent = (String) res[0];
- orc_bleu = (Double) res[1];
- } else {
- HyperGraph hg_oracle = orc_extractor.oracle_extract_hg(hg, hg.sentLen(), lm_order, ref_sent);
- orc_sent = removeSentenceMarkers(getViterbiString(hg_oracle));
- orc_bleu = orc_extractor.get_best_goal_cost(hg, orc_extractor.g_tbl_split_virtual_items);
-
- time_on_orc_extract += System.currentTimeMillis() - start_time;
- System.out.println("num_virtual_items: " + orc_extractor.g_num_virtual_items
- + " num_virtual_dts: " + orc_extractor.g_num_virtual_deductions);
- // System.out.println("oracle extract: " + (System.currentTimeMillis()-start_time));
- }
-
- orc_out.write(orc_sent + "\n");
- System.out.println("orc bleu is " + orc_bleu);
- }
- orc_out.close();
-
- System.out.println("time_on_reading: " + time_on_reading);
- System.out.println("time_on_orc_extract: " + time_on_orc_extract);
- System.out.println("total running time: " + (System.currentTimeMillis() - start_time0));
- }
-
- // find the oracle hypothesis in the nbest list
- public Object[] oracle_extract_nbest(KBestExtractor kbest_extractor, HyperGraph hg, int n,
- boolean do_ngram_clip, String ref_sent) {
- if (hg.goalNode == null)
- return null;
- kbest_extractor.resetState();
- int next_n = 0;
- double orc_bleu = -1;
- String orc_sent = null;
- while (true) {
- String hyp_sent = kbest_extractor.getKthHyp(hg.goalNode, ++next_n);// ?????????
- if (hyp_sent == null || next_n > n)
- break;
- double t_bleu = compute_sentence_bleu(ref_sent, hyp_sent, do_ngram_clip, 4);
- if (t_bleu > orc_bleu) {
- orc_bleu = t_bleu;
- orc_sent = hyp_sent;
- }
- }
- System.out.println("Oracle sent: " + orc_sent);
- System.out.println("Oracle bleu: " + orc_bleu);
- Object[] res = new Object[2];
- res[0] = orc_sent;
- res[1] = orc_bleu;
- return res;
- }
-
- public HyperGraph oracle_extract_hg(HyperGraph hg, int src_sent_len_in, int lm_order,
- String ref_sent_str) {
- int[] ref_sent = Vocabulary.addAll(ref_sent_str);
- g_lm_order = lm_order;
- src_sent_len = src_sent_len_in;
- ref_sent_len = ref_sent.length;
-
- tbl_ref_ngrams.clear();
- get_ngrams(tbl_ref_ngrams, g_bleu_order, ref_sent, false);
- if (using_left_equiv_state || using_right_equiv_state) {
- tbl_prefix.clear();
- tbl_suffix.clear();
- setup_prefix_suffix_tbl(ref_sent, g_bleu_order, tbl_prefix, tbl_suffix);
- setup_prefix_suffix_grammar(ref_sent, g_bleu_order, grammar_prefix, grammar_suffix);// TODO
- }
- split_hg(hg);
-
- // System.out.println("best bleu is " + get_best_goal_cost( hg, g_tbl_split_virtual_items));
- return get_1best_tree_hg(hg, g_tbl_split_virtual_items);
- }
-
- /*
- * This procedure does (1) identify all possible match (2) add a new deduction for each matches
- */
- protected void process_one_combination_axiom(HGNode parent_item,
- HashMap<String, VirtualItem> virtual_item_sigs, HyperEdge cur_dt) {
- if (null == cur_dt.getRule()) {
- throw new RuntimeException("error null rule in axiom");
- }
- double avg_ref_len = (parent_item.j - parent_item.i >= src_sent_len) ? ref_sent_len
- : (parent_item.j - parent_item.i) * ref_sent_len * 1.0 / src_sent_len;// avg len?
- double bleu_score[] = new double[1];
- DPStateOracle dps = compute_state(parent_item, cur_dt, null, tbl_ref_ngrams,
- do_local_ngram_clip, g_lm_order, avg_ref_len, bleu_score, tbl_suffix, tbl_prefix);
- VirtualDeduction t_dt = new VirtualDeduction(cur_dt, null, -bleu_score[0]);// cost: -best_bleu
- g_num_virtual_deductions++;
- add_deduction(parent_item, virtual_item_sigs, t_dt, dps, true);
- }
-
- /*
- * This procedure does (1) create a new deduction (based on cur_dt and ant_virtual_item) (2) find
- * whether an Item can contain this deduction (based on virtual_item_sigs which is a hashmap
- * specific to a parent_item) (2.1) if yes, add the deduction, (2.2) otherwise (2.2.1) create a
- * new item (2.2.2) and add the item into virtual_item_sigs
- */
- protected void process_one_combination_nonaxiom(HGNode parent_item,
- HashMap<String, VirtualItem> virtual_item_sigs, HyperEdge cur_dt,
- ArrayList<VirtualItem> l_ant_virtual_item) {
- if (null == l_ant_virtual_item) {
- throw new RuntimeException("wrong call in process_one_combination_nonaxiom");
- }
- double avg_ref_len = (parent_item.j - parent_item.i >= src_sent_len) ? ref_sent_len
- : (parent_item.j - parent_item.i) * ref_sent_len * 1.0 / src_sent_len;// avg len?
- double bleu_score[] = new double[1];
- DPStateOracle dps = compute_state(parent_item, cur_dt, l_ant_virtual_item, tbl_ref_ngrams,
- do_local_ngram_clip, g_lm_order, avg_ref_len, bleu_score, tbl_suffix, tbl_prefix);
- VirtualDeduction t_dt = new VirtualDeduction(cur_dt, l_ant_virtual_item, -bleu_score[0]);// cost:
- // -best_bleu
- g_num_virtual_deductions++;
- add_deduction(parent_item, virtual_item_sigs, t_dt, dps, true);
- }
-
- // DPState maintain all the state information at an item that is required during dynamic
- // programming
- protected static class DPStateOracle extends DPState {
- int best_len; // this may not be used in the signature
- int[] ngram_matches;
- int[] left_lm_state;
- int[] right_lm_state;
-
- public DPStateOracle(int blen, int[] matches, int[] left, int[] right) {
- best_len = blen;
- ngram_matches = matches;
- left_lm_state = left;
- right_lm_state = right;
- }
-
- protected String get_signature() {
- StringBuffer res = new StringBuffer();
- if (maitain_length_state) {
- res.append(best_len);
- res.append(' ');
- }
- if (null != left_lm_state) { // goal-item have null state
- for (int i = 0; i < left_lm_state.length; i++) {
- res.append(left_lm_state[i]);
- res.append(' ');
- }
- }
- res.append("lzf ");
-
- if (null != right_lm_state) { // goal-item have null state
- for (int i = 0; i < right_lm_state.length; i++) {
- res.append(right_lm_state[i]);
- res.append(' ');
- }
- }
- // if(left_lm_state==null || right_lm_state==null)System.out.println("sig is: " +
- // res.toString());
- return res.toString();
- }
-
- protected void print() {
- StringBuffer res = new StringBuffer();
- res.append("DPstate: best_len: ");
- res.append(best_len);
- for (int i = 0; i < ngram_matches.length; i++) {
- res.append("; ngram: ");
- res.append(ngram_matches[i]);
- }
- System.out.println(res.toString());
- }
- }
-
- // ########################## commmon funcions #####################
- // based on tbl_oracle_states, tbl_ref_ngrams, and dt, get the state
- // get the new state: STATE_BEST_DEDUCT STATE_BEST_BLEU STATE_BEST_LEN NGRAM_MATCH_COUNTS
- protected DPStateOracle compute_state(HGNode parent_item, HyperEdge dt,
- ArrayList<VirtualItem> l_ant_virtual_item, HashMap<String, Integer> tbl_ref_ngrams,
- boolean do_local_ngram_clip, int lm_order, double ref_len, double[] bleu_score,
- HashMap<String, Boolean> tbl_suffix, HashMap<String, Boolean> tbl_prefix) {
- // ##### deductions under "goal item" does not have rule
- if (null == dt.getRule()) {
- if (l_ant_virtual_item.size() != 1) {
- throw new RuntimeException("error deduction under goal item have more than one item");
- }
- bleu_score[0] = -l_ant_virtual_item.get(0).best_virtual_deduction.best_cost;
- return new DPStateOracle(0, null, null, null); // no DPState at all
- }
-
- // ################## deductions *not* under "goal item"
- HashMap<String, Integer> new_ngram_counts = new HashMap<String, Integer>();// new ngrams created
- // due to the
- // combination
- HashMap<String, Integer> old_ngram_counts = new HashMap<String, Integer>();// the ngram that has
- // already been
- // computed
- int total_hyp_len = 0;
- int[] num_ngram_match = new int[g_bleu_order];
- int[] en_words = dt.getRule().getEnglish();
-
- // ####calulate new and old ngram counts, and len
-
- ArrayList<Integer> words = new ArrayList<Integer>();
-
- // used for compute left- and right- lm state
- ArrayList<Integer> left_state_sequence = null;
- // used for compute left- and right- lm state
- ArrayList<Integer> right_state_sequence = null;
-
- int correct_lm_order = lm_order;
- if (always_maintain_seperate_lm_state || lm_order < g_bleu_order) {
- left_state_sequence = new ArrayList<Integer>();
- right_state_sequence = new ArrayList<Integer>();
- correct_lm_order = g_bleu_order; // if lm_order is smaller than g_bleu_order, we will get the
- // lm state by ourself
- }
-
- // #### get left_state_sequence, right_state_sequence, total_hyp_len, num_ngram_match
- for (int c = 0; c < en_words.length; c++) {
- int c_id = en_words[c];
- if (FormatUtils.isNonterminal(c_id)) {
- int index = -(c_id + 1);
- DPStateOracle ant_state = (DPStateOracle) l_ant_virtual_item.get(index).dp_state;
- total_hyp_len += ant_state.best_len;
- for (int t = 0; t < g_bleu_order; t++) {
- num_ngram_match[t] += ant_state.ngram_matches[t];
- }
- int[] l_context = ant_state.left_lm_state;
- int[] r_context = ant_state.right_lm_state;
- for (int t : l_context) { // always have l_context
- words.add(t);
- if (null != left_state_sequence && left_state_sequence.size() < g_bleu_order - 1) {
- left_state_sequence.add(t);
- }
- }
- get_ngrams(old_ngram_counts, g_bleu_order, l_context, true);
- if (r_context.length >= correct_lm_order - 1) { // the right and left are NOT overlapping
- get_ngrams(new_ngram_counts, g_bleu_order, words, true);
- get_ngrams(old_ngram_counts, g_bleu_order, r_context, true);
- words.clear();// start a new chunk
- if (null != right_state_sequence) {
- right_state_sequence.clear();
- }
- for (int t : r_context) {
- words.add(t);
- }
- }
- if (null != right_state_sequence) {
- for (int t : r_context) {
- right_state_sequence.add(t);
- }
- }
- } else {
- words.add(c_id);
- total_hyp_len += 1;
- if (null != left_state_sequence && left_state_sequence.size() < g_bleu_order - 1) {
- left_state_sequence.add(c_id);
- }
- if (null != right_state_sequence) {
- right_state_sequence.add(c_id);
- }
- }
- }
- get_ngrams(new_ngram_counts, g_bleu_order, words, true);
-
- // ####now deduct ngram counts
- for (String ngram : new_ngram_counts.keySet()) {
- if (tbl_ref_ngrams.containsKey(ngram)) {
- int final_count = (Integer) new_ngram_counts.get(ngram);
- if (old_ngram_counts.containsKey(ngram)) {
- final_count -= (Integer) old_ngram_counts.get(ngram);
- // BUG: Whoa, is that an actual hard-coded ID in there? :)
- if (final_count < 0) {
- throw new RuntimeException("negative count for ngram: " + Vocabulary.word(11844)
- + "; new: " + new_ngram_counts.get(ngram) + "; old: " + old_ngram_counts.get(ngram));
- }
- }
- if (final_count > 0) { // TODO: not correct/global ngram clip
- if (do_local_ngram_clip) {
- // BUG: use joshua.util.Regex.spaces.split(...)
- num_ngram_match[ngram.split("\\s+").length - 1] += Support.findMin(final_count,
- (Integer) tbl_ref_ngrams.get(ngram));
- } else {
- // BUG: use joshua.util.Regex.spaces.split(...)
- num_ngram_match[ngram.split("\\s+").length - 1] += final_count; // do not do any cliping
- }
- }
- }
- }
-
- // ####now calculate the BLEU score and state
- int[] left_lm_state = null;
- int[] right_lm_state = null;
- left_lm_state = get_left_equiv_state(left_state_sequence, tbl_suffix);
- right_lm_state = get_right_equiv_state(right_state_sequence, tbl_prefix);
-
- // debug
- // System.out.println("lm_order is " + lm_order);
- // compare_two_int_arrays(left_lm_state,
- // (int[])parent_item.tbl_states.get(Symbol.LM_L_STATE_SYM_ID));
- // compare_two_int_arrays(right_lm_state,
- // (int[])parent_item.tbl_states.get(Symbol.LM_R_STATE_SYM_ID));
- // end
-
- bleu_score[0] = compute_bleu(total_hyp_len, ref_len, num_ngram_match, g_bleu_order);
- // System.out.println("blue score is " + bleu_score[0]);
- return new DPStateOracle(total_hyp_len, num_ngram_match, left_lm_state, right_lm_state);
- }
-
- private int[] get_left_equiv_state(ArrayList<Integer> left_state_sequence,
- HashMap<String, Boolean> tbl_suffix) {
- int l_size = (left_state_sequence.size() < g_bleu_order - 1) ? left_state_sequence.size()
- : (g_bleu_order - 1);
- int[] left_lm_state = new int[l_size];
- if (!using_left_equiv_state || l_size < g_bleu_order - 1) { // regular
- for (int i = 0; i < l_size; i++) {
- left_lm_state[i] = left_state_sequence.get(i);
- }
- } else {
- for (int i = l_size - 1; i >= 0; i--) { // right to left
- if (is_a_suffix_in_tbl(left_state_sequence, 0, i, tbl_suffix)) {
- // if(is_a_suffix_in_grammar(left_state_sequence, 0, i, grammar_suffix)){
- for (int j = i; j >= 0; j--) {
- left_lm_state[j] = left_state_sequence.get(j);
- }
- break;
- } else {
- left_lm_state[i] = this.NULL_LEFT_LM_STATE_SYM_ID;
- }
- }
- // System.out.println("origi left:" + Symbol.get_string(left_state_sequence) + "; equiv left:"
- // + Symbol.get_string(left_lm_state));
- }
- return left_lm_state;
- }
-
- private boolean is_a_suffix_in_tbl(ArrayList<Integer> left_state_sequence, int start_pos,
- int end_pos, HashMap<String, Boolean> tbl_suffix) {
- if ((Integer) left_state_sequence.get(end_pos) == this.NULL_LEFT_LM_STATE_SYM_ID) {
- return false;
- }
- StringBuffer suffix = new StringBuffer();
- for (int i = end_pos; i >= start_pos; i--) { // right-most first
- suffix.append(left_state_sequence.get(i));
- if (i > start_pos)
- suffix.append(' ');
- }
- return (Boolean) tbl_suffix.containsKey(suffix.toString());
- }
-
- private int[] get_right_equiv_state(ArrayList<Integer> right_state_sequence,
- HashMap<String, Boolean> tbl_prefix) {
- int r_size = (right_state_sequence.size() < g_bleu_order - 1) ? right_state_sequence.size()
- : (g_bleu_order - 1);
- int[] right_lm_state = new int[r_size];
- if (!using_right_equiv_state || r_size < g_bleu_order - 1) { // regular
- for (int i = 0; i < r_size; i++) {
- right_lm_state[i] = (Integer) right_state_sequence.get(right_state_sequence.size() - r_size
- + i);
- }
- } else {
- for (int i = 0; i < r_size; i++) { // left to right
- if (is_a_prefix_in_tbl(right_state_sequence, right_state_sequence.size() - r_size + i,
- right_state_sequence.size() - 1, tbl_prefix)) {
- // if(is_a_prefix_in_grammar(right_state_sequence, right_state_sequence.size()-r_size+i,
- // right_state_sequence.size()-1, grammar_prefix)){
- for (int j = i; j < r_size; j++) {
- right_lm_state[j] = (Integer) right_state_sequence.get(right_state_sequence.size()
- - r_size + j);
- }
- break;
- } else {
- right_lm_state[i] = this.NULL_RIGHT_LM_STATE_SYM_ID;
- }
- }
- // System.out.println("origi right:" + Symbol.get_string(right_state_sequence)+
- // "; equiv right:" + Symbol.get_string(right_lm_state));
- }
- return right_lm_state;
- }
-
- private boolean is_a_prefix_in_tbl(ArrayList<Integer> right_state_sequence, int start_pos,
- int end_pos, HashMap<String, Boolean> tbl_prefix) {
- if (right_state_sequence.get(start_pos) == this.NULL_RIGHT_LM_STATE_SYM_ID) {
- return false;
- }
- StringBuffer prefix = new StringBuffer();
- for (int i = start_pos; i <= end_pos; i++) {
- prefix.append(right_state_sequence.get(i));
- if (i < end_pos)
- prefix.append(' ');
- }
- return (Boolean) tbl_prefix.containsKey(prefix.toString());
- }
-
- public static void compare_two_int_arrays(int[] a, int[] b) {
- if (a.length != b.length) {
- throw new RuntimeException("two arrays do not have same size");
- }
- for (int i = 0; i < a.length; i++) {
- if (a[i] != b[i]) {
- throw new RuntimeException("elements in two arrays are not same");
- }
- }
- }
-
- // sentence-bleu: BLEU= bp * prec; where prec = exp (sum 1/4 * log(prec[order]))
- public static double compute_bleu(int hyp_len, double ref_len, int[] num_ngram_match,
- int bleu_order) {
- if (hyp_len <= 0 || ref_len <= 0) {
- throw new RuntimeException("ref or hyp is zero len");
- }
- double res = 0;
- double wt = 1.0 / bleu_order;
- double prec = 0;
- double smooth_factor = 1.0;
- for (int t = 0; t < bleu_order && t < hyp_len; t++) {
- if (num_ngram_match[t] > 0) {
- prec += wt * Math.log(num_ngram_match[t] * 1.0 / (hyp_len - t));
- } else {
- smooth_factor *= 0.5;// TODO
- prec += wt * Math.log(smooth_factor / (hyp_len - t));
- }
- }
- double bp = (hyp_len >= ref_len) ? 1.0 : Math.exp(1 - ref_len / hyp_len);
- res = bp * Math.exp(prec);
- // System.out.println("hyp_len: " + hyp_len + "; ref_len:" + ref_len + "prec: " + Math.exp(prec)
- // + "; bp: " + bp + "; bleu: " + res);
- return res;
- }
-
- // accumulate ngram counts into tbl
- public void get_ngrams(HashMap<String, Integer> tbl, int order, int[] wrds,
- boolean ignore_null_equiv_symbol) {
- for (int i = 0; i < wrds.length; i++) {
- for (int j = 0; j < order && j + i < wrds.length; j++) { // ngram: [i,i+j]
- boolean contain_null = false;
- StringBuffer ngram = new StringBuffer();
- for (int k = i; k <= i + j; k++) {
- if (wrds[k] == this.NULL_LEFT_LM_STATE_SYM_ID
- || wrds[k] == this.NULL_RIGHT_LM_STATE_SYM_ID) {
- contain_null = true;
- if (ignore_null_equiv_symbol)
- break;
- }
- ngram.append(wrds[k]);
- if (k < i + j)
- ngram.append(' ');
- }
- if (ignore_null_equiv_symbol && contain_null)
- continue; // skip this ngram
- String ngram_str = ngram.toString();
- if (tbl.containsKey(ngram_str)) {
- tbl.put(ngram_str, (Integer) tbl.get(ngram_str) + 1);
- } else {
- tbl.put(ngram_str, 1);
- }
- }
- }
- }
-
- /** accumulate ngram counts into tbl. */
- public void get_ngrams(HashMap<String, Integer> tbl, int order, ArrayList<Integer> wrds,
- boolean ignore_null_equiv_symbol) {
- for (int i = 0; i < wrds.size(); i++) {
- // ngram: [i,i+j]
- for (int j = 0; j < order && j + i < wrds.size(); j++) {
- boolean contain_null = false;
- StringBuffer ngram = new StringBuffer();
- for (int k = i; k <= i + j; k++) {
- int t_wrd = (Integer) wrds.get(k);
- if (t_wrd == this.NULL_LEFT_LM_STATE_SYM_ID || t_wrd == this.NULL_RIGHT_LM_STATE_SYM_ID) {
- contain_null = true;
- if (ignore_null_equiv_symbol)
- break;
- }
- ngram.append(t_wrd);
- if (k < i + j)
- ngram.append(' ');
- }
- // skip this ngram
- if (ignore_null_equiv_symbol && contain_null)
- continue;
-
- String ngram_str = ngram.toString();
- if (tbl.containsKey(ngram_str)) {
- tbl.put(ngram_str, (Integer) tbl.get(ngram_str) + 1);
- } else {
- tbl.put(ngram_str, 1);
- }
- }
- }
- }
-
- // do_ngram_clip: consider global n-gram clip
- public double compute_sentence_bleu(String ref_sent, String hyp_sent, boolean do_ngram_clip,
- int bleu_order) {
- // BUG: use joshua.util.Regex.spaces.split(...)
- int[] numeric_ref_sent = Vocabulary.addAll(ref_sent);
- int[] numeric_hyp_sent = Vocabulary.addAll(hyp_sent);
- return compute_sentence_bleu(numeric_ref_sent, numeric_hyp_sent, do_ngram_clip, bleu_order);
- }
-
- public double compute_sentence_bleu(int[] ref_sent, int[] hyp_sent, boolean do_ngram_clip,
- int bleu_order) {
- double res_bleu = 0;
- int order = 4;
- HashMap<String, Integer> ref_ngram_tbl = new HashMap<String, Integer>();
- get_ngrams(ref_ngram_tbl, order, ref_sent, false);
- HashMap<String, Integer> hyp_ngram_tbl = new HashMap<String, Integer>();
- get_ngrams(hyp_ngram_tbl, order, hyp_sent, false);
-
- int[] num_ngram_match = new int[order];
- for (String ngram : hyp_ngram_tbl.keySet()) {
- if (ref_ngram_tbl.containsKey(ngram)) {
- if (do_ngram_clip) {
- // BUG: use joshua.util.Regex.spaces.split(...)
- num_ngram_match[ngram.split("\\s+").length - 1] += Support.findMin(
- (Integer) ref_ngram_tbl.get(ngram), (Integer) hyp_ngram_tbl.get(ngram)); // ngram clip
- } else {
- // BUG: use joshua.util.Regex.spaces.split(...)
- num_ngram_match[ngram.split("\\s+").length - 1] += (Integer) hyp_ngram_tbl.get(ngram);// without
- // ngram
- // count
- // clipping
- }
- }
- }
- res_bleu = compute_bleu(hyp_sent.length, ref_sent.length, num_ngram_match, bleu_order);
- // System.out.println("hyp_len: " + hyp_sent.length + "; ref_len:" + ref_sent.length +
- // "; bleu: " + res_bleu +" num_ngram_matches: " + num_ngram_match[0] + " " +num_ngram_match[1]+
- // " " + num_ngram_match[2] + " " +num_ngram_match[3]);
-
- return res_bleu;
- }
-
- // #### equivalent lm stuff ############
- public static void setup_prefix_suffix_tbl(int[] wrds, int order,
- HashMap<String, Boolean> prefix_tbl, HashMap<String, Boolean> suffix_tbl) {
- for (int i = 0; i < wrds.length; i++) {
- for (int j = 0; j < order && j + i < wrds.length; j++) { // ngram: [i,i+j]
- StringBuffer ngram = new StringBuffer();
- // ### prefix
- for (int k = i; k < i + j; k++) { // all ngrams [i,i+j-1]
- ngram.append(wrds[k]);
- prefix_tbl.put(ngram.toString(), true);
- ngram.append(' ');
- }
- // ### suffix: right-most wrd first
- ngram = new StringBuffer();
- for (int k = i + j; k > i; k--) { // all ngrams [i+1,i+j]: reverse order
- ngram.append(wrds[k]);
- suffix_tbl.put(ngram.toString(), true);// stored in reverse order
- ngram.append(' ');
- }
- }
- }
- }
-
- // #### equivalent lm stuff ############
- public static void setup_prefix_suffix_grammar(int[] wrds, int order, PrefixGrammar prefix_gr,
- PrefixGrammar suffix_gr) {
- for (int i = 0; i < wrds.length; i++) {
- for (int j = 0; j < order && j + i < wrds.length; j++) { // ngram: [i,i+j]
- // ### prefix
- prefix_gr.add_ngram(wrds, i, i + j - 1);// ngram: [i,i+j-1]
-
- // ### suffix: right-most wrd first
- int[] reverse_wrds = new int[j];
- for (int k = i + j, t = 0; k > i; k--) { // all ngrams [i+1,i+j]: reverse order
- reverse_wrds[t++] = wrds[k];
- }
- suffix_gr.add_ngram(reverse_wrds, 0, j - 1);
- }
- }
- }
-
- /*
- * a backoff node is a hashtable, it may include: (1) probabilititis for next words (2) pointers
- * to a next-layer backoff node (hashtable) (3) backoff weight for this node (4) suffix/prefix
- * flag to indicate that there is ngrams start from this suffix
- */
- private static class PrefixGrammar {
-
- private static class PrefixGrammarNode extends HashMap<Integer, PrefixGrammarNode> {
- private static final long serialVersionUID = 1L;
- };
-
- PrefixGrammarNode root = new PrefixGrammarNode();
-
- // add prefix information
- public void add_ngram(int[] wrds, int start_pos, int end_pos) {
- // ######### identify the position, and insert the trinodes if necessary
- PrefixGrammarNode pos = root;
- for (int k = start_pos; k <= end_pos; k++) {
- int cur_sym_id = wrds[k];
- PrefixGrammarNode next_layer = pos.get(cur_sym_id);
-
- if (null != next_layer) {
- pos = next_layer;
- } else {
- // next layer node
- PrefixGrammarNode tmp = new PrefixGrammarNode();
- pos.put(cur_sym_id, tmp);
- pos = tmp;
- }
- }
- }
-
- @SuppressWarnings("unused")
- public boolean contain_ngram(ArrayList<Integer> wrds, int start_pos, int end_pos) {
- if (end_pos < start_pos)
- return false;
- PrefixGrammarNode pos = root;
- for (int k = start_pos; k <= end_pos; k++) {
- int cur_sym_id = wrds.get(k);
- PrefixGrammarNode next_layer = pos.get(cur_sym_id);
- if (next_layer != null) {
- pos = next_layer;
- } else {
- return false;
- }
- }
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/f2f82c38/src/joshua/oracle/OracleExtractor.java
----------------------------------------------------------------------
diff --git a/src/joshua/oracle/OracleExtractor.java b/src/joshua/oracle/OracleExtractor.java
deleted file mode 100644
index d4a0019..0000000
--- a/src/joshua/oracle/OracleExtractor.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.oracle;
-
-import joshua.decoder.hypergraph.HyperGraph;
-
-/**
- * Convenience wrapper class for oracle extraction code.
- *
- * @author Lane Schwartz
- */
-public class OracleExtractor {
-
- private final OracleExtractionHG extractor;
-
- /**
- * Constructs an object capable of extracting an oracle hypergraph.
- */
- public OracleExtractor() {
-
- int baselineLanguageModelFeatureID = 0;
- this.extractor = new OracleExtractionHG(baselineLanguageModelFeatureID);
-
- }
-
- /**
- * Extract a hypergraph that represents the translation from the original shared forest hypergraph
- * that is closest to the reference translation.
- *
- * @param forest Original hypergraph representing a shared forest.
- * @param lmOrder N-gram order of the language model.
- * @param reference Reference sentence.
- * @return Hypergraph closest to the reference.
- */
- public HyperGraph getOracle(HyperGraph forest, int lmOrder, String reference) {
- if (reference != null)
- return extractor.oracle_extract_hg(forest, forest.sentLen(), lmOrder, reference);
-
- return null;
- }
-
-}