[13/32] incubator-joshua git commit: Removed call to Moses phrase table conversion script; phrase decoding now works (not backwards combat!)

[mj...@apache.org]

Removed call to Moses phrase table conversion script; phrase decoding now works (not backwards combat!)


Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/a43f7353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/a43f7353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/a43f7353

Branch: refs/heads/JOSHUA-252
Commit: a43f7353e78ccbedabcf144dad33c57e864bbf84
Parents: f5adcde
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed May 25 09:14:23 2016 +0200
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Wed May 25 09:14:23 2016 +0200

----------------------------------------------------------------------
 scripts/support/grammar-packer.pl         |  2 +-
 scripts/support/moses_phrase_to_joshua.pl | 23 -----------------------
 2 files changed, 1 insertion(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a43f7353/scripts/support/grammar-packer.pl
----------------------------------------------------------------------
diff --git a/scripts/support/grammar-packer.pl b/scripts/support/grammar-packer.pl
index d674d75..7cd3153 100755
--- a/scripts/support/grammar-packer.pl
+++ b/scripts/support/grammar-packer.pl
@@ -76,7 +76,7 @@ foreach my $grammar (@grammars) {
     }
   } else {
     # Moses phrase-based grammar -- prepend nonterminal symbol and -log() the weights
-    if (system("$CAT $grammar | $JOSHUA/scripts/support/moses_phrase_to_joshua.pl | sed 's/ ||| /\t/g' | LC_ALL=C sort -t'\t' -k2,2 -k3,3 --buffer-size=$opts{m} -T $opts{T} | sed 's/\t/ ||| /g' | gzip -9n > $sorted_grammar")) {
+    if (system("$CAT $grammar | sed 's/ ||| /\t/g' | LC_ALL=C sort -t'\t' -k1,1 -k2,2 --buffer-size=$opts{m} -T $opts{T} | sed 's/\t/ ||| /g' | gzip -9n > $sorted_grammar")) {
       print STDERR "* FATAL: Couldn't sort the grammar (not enough memory? short on tmp space?)\n";
       exit 2;
     }

http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a43f7353/scripts/support/moses_phrase_to_joshua.pl
----------------------------------------------------------------------
diff --git a/scripts/support/moses_phrase_to_joshua.pl b/scripts/support/moses_phrase_to_joshua.pl
deleted file mode 100755
index 9fe727e..0000000
--- a/scripts/support/moses_phrase_to_joshua.pl
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/perl
-
-# Converts a Moses phrase table to a Joshua grammar, suitable for packing.
-# (Joshua can read in Moses phrase tables directly when using the in-memory
-# representation, so in that case there is no need to do the conversion).
-
-use strict;
-use warnings;
-
-while (my $line = <>) {
-  chomp($line);
-  my @tokens = split(/ \|\|\| /, $line);
-
-  unshift(@tokens, "[X]");
-  $tokens[3] = join(" ", map { -mylog($_) } split(' ', $tokens[3]));
-
-  print join(" ||| ", @tokens) . $/;
-}
-
-sub mylog {
-  my ($num) = @_;
-  return ($num == 0) ? -100 : log($num);
-}