You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/15 14:51:18 UTC
incubator-joshua git commit: Fixed NumTranslationOptions test and
added deterministic output order of feature vectors
Repository: incubator-joshua
Updated Branches:
refs/heads/7 7e7baaffc -> bc050e3b5
Fixed NumTranslationOptions test and added deterministic output order of feature vectors
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/bc050e3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/bc050e3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/bc050e3b
Branch: refs/heads/7
Commit: bc050e3b5c99f71c4d5945dc9852a13e825958f7
Parents: 7e7baaf
Author: Hieber, Felix <fh...@amazon.de>
Authored: Thu Sep 15 16:48:58 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Thu Sep 15 16:48:58 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/ff/FeatureVector.java | 4 +++-
.../num_translation_options/grammar.packed/config | 2 ++
.../grammar.packed/slice_00000.alignments | Bin 0 -> 33 bytes
.../grammar.packed/slice_00000.features | Bin 43 -> 43 bytes
.../grammar.packed/slice_00000.source | Bin 132 -> 132 bytes
.../grammar.packed/slice_00000.target | Bin 120 -> 120 bytes
.../grammar.packed/vocabulary | Bin 144 -> 137 bytes
.../num_translation_options/joshua-packed.config | 4 ++--
.../decoder/num_translation_options/joshua.config | 4 ++--
.../num_translation_options/joshua.config.packed | 4 ++--
.../output-no-dot-chart.gold | 8 ++++----
.../num_translation_options/output-packed.gold | 8 ++++----
.../decoder/num_translation_options/output.gold | 8 ++++----
13 files changed, 23 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
index 7be7317..813c9ff 100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
@@ -22,6 +22,7 @@ import static java.util.stream.Collectors.joining;
import java.util.AbstractMap;
import java.util.Collection;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -141,7 +142,8 @@ public class FeatureVector extends AbstractMap<Integer, Float> {
public String textFormat() {
return internalMap.entrySet()
.stream()
- .map(e -> String.format("%s=%.6f", FeatureMap.getFeature(e.getKey()), e.getValue()) )
+ .map(e -> String.format("%s=%.6f", FeatureMap.getFeature(e.getKey()), e.getValue()))
+ .sorted(Comparator.reverseOrder())
.collect(joining(" "));
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/config b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/config
new file mode 100644
index 0000000..fde9f40
--- /dev/null
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/config
@@ -0,0 +1,2 @@
+version = 4
+max-source-len = 4
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.alignments
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.alignments b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.alignments
new file mode 100644
index 0000000..f5241c9
Binary files /dev/null and b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.alignments differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.features
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.features b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.features
index bb6eaa6..4aa11e0 100644
Binary files a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.features and b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.features differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.source
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.source b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.source
index 135f4af..2b3fd35 100644
Binary files a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.source and b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.source differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.target
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.target b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.target
index 776a21e..a6087c3 100644
Binary files a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.target and b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/slice_00000.target differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/vocabulary
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/vocabulary b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/vocabulary
index 47fd1a4..4ae82c4 100644
Binary files a/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/vocabulary and b/joshua-core/src/test/resources/decoder/num_translation_options/grammar.packed/vocabulary differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/joshua-packed.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/joshua-packed.config b/joshua-core/src/test/resources/decoder/num_translation_options/joshua-packed.config
index 681c4d9..8e5019d 100644
--- a/joshua-core/src/test/resources/decoder/num_translation_options/joshua-packed.config
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/joshua-packed.config
@@ -24,7 +24,7 @@ feature-function = OOVPenalty
lm_0 1.2373676802179452
-tm_pt_0 1
-tm_glue_0 1
+pt_0 -1
+glue_0 -1
WordPenalty 1
OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config b/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config
index 88b0290..98aff65 100644
--- a/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config
@@ -24,7 +24,7 @@ feature-function = OOVPenalty
lm_0 1.2373676802179452
-tm_pt_0 1
-tm_glue_0 1
+pt_0 -1
+glue_0 -1
WordPenalty 1
OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config.packed
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config.packed b/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config.packed
index 2d52db2..2a25146 100644
--- a/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config.packed
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/joshua.config.packed
@@ -24,7 +24,7 @@ feature-function = OOVPenalty
lm_0 1.2373676802179452
-tm_pt_0 1
-tm_glue_0 1
+pt_0 -1
+glue_0 -1
WordPenalty 1
OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold b/joshua-core/src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold
index 686122c..d130edd 100644
--- a/joshua-core/src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold
@@ -1,4 +1,4 @@
--19.196 ||| i like taco bell ||| tm_pt_0=4.000 tm_glue_0=1.000 lm_0=-17.449 WordPenalty=-2.606 OOVPenalty=0.000
--19.733 ||| i love taco bell ||| tm_pt_0=5.000 tm_glue_0=1.000 lm_0=-18.690 WordPenalty=-2.606 OOVPenalty=0.000
--22.883 ||| i appreciate taco bell ||| tm_pt_0=3.000 tm_glue_0=1.000 lm_0=-19.620 WordPenalty=-2.606 OOVPenalty=0.000
--424.954 ||| yo quiero taco bell ||| tm_pt_0=0.000 tm_glue_0=4.000 lm_0=-21.293 WordPenalty=-2.606 OOVPenalty=-400.000
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/output-packed.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/output-packed.gold b/joshua-core/src/test/resources/decoder/num_translation_options/output-packed.gold
index 686122c..d9e97f5 100644
--- a/joshua-core/src/test/resources/decoder/num_translation_options/output-packed.gold
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/output-packed.gold
@@ -1,4 +1,4 @@
--19.196 ||| i like taco bell ||| tm_pt_0=4.000 tm_glue_0=1.000 lm_0=-17.449 WordPenalty=-2.606 OOVPenalty=0.000
--19.733 ||| i love taco bell ||| tm_pt_0=5.000 tm_glue_0=1.000 lm_0=-18.690 WordPenalty=-2.606 OOVPenalty=0.000
--22.883 ||| i appreciate taco bell ||| tm_pt_0=3.000 tm_glue_0=1.000 lm_0=-19.620 WordPenalty=-2.606 OOVPenalty=0.000
--424.954 ||| yo quiero taco bell ||| tm_pt_0=0.000 tm_glue_0=4.000 lm_0=-21.293 WordPenalty=-2.606 OOVPenalty=-400.000
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/bc050e3b/joshua-core/src/test/resources/decoder/num_translation_options/output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/num_translation_options/output.gold b/joshua-core/src/test/resources/decoder/num_translation_options/output.gold
index 686122c..d9e97f5 100644
--- a/joshua-core/src/test/resources/decoder/num_translation_options/output.gold
+++ b/joshua-core/src/test/resources/decoder/num_translation_options/output.gold
@@ -1,4 +1,4 @@
--19.196 ||| i like taco bell ||| tm_pt_0=4.000 tm_glue_0=1.000 lm_0=-17.449 WordPenalty=-2.606 OOVPenalty=0.000
--19.733 ||| i love taco bell ||| tm_pt_0=5.000 tm_glue_0=1.000 lm_0=-18.690 WordPenalty=-2.606 OOVPenalty=0.000
--22.883 ||| i appreciate taco bell ||| tm_pt_0=3.000 tm_glue_0=1.000 lm_0=-19.620 WordPenalty=-2.606 OOVPenalty=0.000
--424.954 ||| yo quiero taco bell ||| tm_pt_0=0.000 tm_glue_0=4.000 lm_0=-21.293 WordPenalty=-2.606 OOVPenalty=-400.000
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
\ No newline at end of file