You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/14 13:31:00 UTC
[3/4] incubator-joshua git commit: Moved regression test
decoder/constrained to unit test. Cleaned up the corresponding directory.
Moved regression test decoder/constrained to unit test. Cleaned up the corresponding directory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/99c791d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/99c791d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/99c791d1
Branch: refs/heads/master
Commit: 99c791d1d54702061aafeb7678e88ae40521b419
Parents: 0871742
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Wed Sep 14 15:12:46 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Wed Sep 14 15:12:46 2016 +0200
----------------------------------------------------------------------
.../joshua/decoder/cky/ConstrainedTest.java | 65 ++++++++++++++++++++
.../resources/decoder/constrained/gold.scores | 27 --------
.../resources/decoder/constrained/joshua.config | 8 +--
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 60 +++++++++---------
src/test/resources/decoder/constrained/test.sh | 30 ---------
src/test/resources/decoder/constrained/weights | 22 -------
7 files changed, 99 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.java b/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.java
new file mode 100644
index 0000000..53bab7a
--- /dev/null
+++ b/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.cky;
+
+import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
+import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
+import static org.testng.Assert.assertEquals;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.io.KenLmTestUtil;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+public class ConstrainedTest {
+
+ private JoshuaConfiguration joshuaConfig;
+ private Decoder decoder;
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ if(decoder != null) {
+ decoder.cleanUp();
+ decoder = null;
+ }
+ }
+
+ @Test
+ public void givenInput_whenConstrainedDecoding_thenScoreAndTranslationCorrect() throws Exception {
+ // Given
+ List<String> inputStrings = loadStringsFromFile("src/test/resources/decoder/constrained/input.bn");
+
+ // When
+ configureDecoder("src/test/resources/decoder/constrained/joshua.config");
+ List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
+
+ // Then
+ List<String> goldStrings = loadStringsFromFile("src/test/resources/decoder/constrained/output.gold");
+ assertEquals(decodedStrings, goldStrings);
+ }
+
+ public void configureDecoder(String pathToConfig) throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.readConfigFile(pathToConfig);
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig, ""));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/resources/decoder/constrained/gold.scores
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/constrained/gold.scores b/src/test/resources/decoder/constrained/gold.scores
deleted file mode 100644
index 4b90eb1..0000000
--- a/src/test/resources/decoder/constrained/gold.scores
+++ /dev/null
@@ -1,27 +0,0 @@
- mathematics so science language . ||| -10.471
- mathematics is science language . ||| -10.543
- mathematics that science language . ||| -11.587
- science mathematics that language . ||| -12.065
- mathematics so that the science language . ||| -12.259
- mathematics hence science language . ||| -12.352
- mathematics that is science language . ||| -12.590
- mathematics , science language . ||| -12.619
- mathematics that is why science language . ||| -12.751
- mathematics in science language . ||| -12.755
- mathematics is science language . ||| -10.543
- rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -226.523
- rabindranath 's birth in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.201
- rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.217
- rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.417
- rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.707
- rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.000
- rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.133
- rabindranath was born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.160
- rabindranath born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.266
- rabindranath 's birth in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.386
- rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -226.302
- rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -226.523
- rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.707
- rabindranath born in in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -232.123
- rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.643
- rabindranath born in kolkata was one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -229.108
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/resources/decoder/constrained/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/constrained/joshua.config b/src/test/resources/decoder/constrained/joshua.config
index f12804c..635f02b 100644
--- a/src/test/resources/decoder/constrained/joshua.config
+++ b/src/test/resources/decoder/constrained/joshua.config
@@ -1,7 +1,7 @@
-lm = kenlm 5 false false 100 lm.gz
+lm = kenlm 5 false false 100 src/test/resources/decoder/constrained/lm.gz
-tm = thrax pt 12 grammar.gz
-tm = thrax glue -1 glue-grammar
+tm = thrax pt 12 src/test/resources/decoder/constrained/grammar.gz
+tm = thrax glue -1 src/test/resources/decoder/constrained/glue-grammar
mark_oovs = false
@@ -11,7 +11,7 @@ goalSymbol = GOAL
#pruning config
pop-limit = 100
-output-format = %i %c %s
+output-format = %c %s
#nbest config
use_unique_nbest = true
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/resources/decoder/constrained/output.bleu
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/constrained/output.bleu b/src/test/resources/decoder/constrained/output.bleu
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/resources/decoder/constrained/output.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/constrained/output.gold b/src/test/resources/decoder/constrained/output.gold
index b64847e..a48d5cb 100644
--- a/src/test/resources/decoder/constrained/output.gold
+++ b/src/test/resources/decoder/constrained/output.gold
@@ -1,30 +1,30 @@
-0 -10.471 mathematics so science language .
-0 -10.543 mathematics is science language .
-0 -11.587 mathematics that science language .
-0 -12.065 science mathematics that language .
-0 -12.259 mathematics so that the science language .
-0 -12.352 mathematics hence science language .
-0 -12.590 mathematics that is science language .
-0 -12.619 mathematics , science language .
-0 -12.751 mathematics that is why science language .
-0 -12.755 mathematics in science language .
-1 -10.543 mathematics is science language .
-2 -226.302 rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -226.523 rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -226.884 rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -227.201 rabindranath 's birth in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -227.217 rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -227.707 rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -227.982 rabindranath was born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -228.069 rabindranath was born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -228.133 rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-2 -228.160 rabindranath was born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-3 -226.302 rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-4 -226.523 rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-4 -230.920 rabindranath born in in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-5 -227.707 rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-5 -232.123 rabindranath born in in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-6 -226.884 rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-6 -231.320 rabindranath was born kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-6 -232.836 rabindranath was born in in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
-7 -229.108 rabindranath born in kolkata was one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-10.471 mathematics so science language .
+-10.543 mathematics is science language .
+-11.587 mathematics that science language .
+-12.065 science mathematics that language .
+-12.259 mathematics so that the science language .
+-12.352 mathematics hence science language .
+-12.590 mathematics that is science language .
+-12.619 mathematics , science language .
+-12.751 mathematics that is why science language .
+-12.755 mathematics in science language .
+-10.543 mathematics is science language .
+-226.302 rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.523 rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.884 rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.201 rabindranath 's birth in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.217 rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.707 rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.982 rabindranath was born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-228.069 rabindranath was born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-228.133 rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-228.160 rabindranath was born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.302 rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.523 rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-230.920 rabindranath born in in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.707 rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-232.123 rabindranath born in in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.884 rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-231.320 rabindranath was born kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-232.836 rabindranath was born in in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-229.108 rabindranath born in kolkata was one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/resources/decoder/constrained/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/constrained/test.sh b/src/test/resources/decoder/constrained/test.sh
deleted file mode 100755
index b21e2ff..0000000
--- a/src/test/resources/decoder/constrained/test.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -u
-
-cat input.bn | $JOSHUA/bin/joshua-decoder -m 1g -threads 1 -c joshua.config > output 2> log
-
-# Compare
-diff -u output output.gold > diff
-
-if [ $? -eq 0 ]; then
- rm -f diff log output output.scores
- exit 0
-else
- exit 1
-fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/99c791d1/src/test/resources/decoder/constrained/weights
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/constrained/weights b/src/test/resources/decoder/constrained/weights
deleted file mode 100644
index c4e32e7..0000000
--- a/src/test/resources/decoder/constrained/weights
+++ /dev/null
@@ -1,22 +0,0 @@
-lm_0 1.2373676802179452
-
-tm_pt_0 -2.4497429277910214
-tm_pt_1 0.7224581556224123
-tm_pt_2 -0.31689069155153504
-tm_pt_3 0.33861043967238036
-tm_pt_4 0.03553113401320236
-tm_pt_5 0.19138972284064748
-tm_pt_6 0.3417994095521415
-tm_pt_7 -0.9936312455671283
-tm_pt_8 0.9070737587091975
-tm_pt_9 0.8202511858619419
-tm_pt_10 0.2593091306160006
-tm_pt_11 0.25597137004462134
-tm_pt_12 0.3538894647790496
-tm_pt_13 -0.36212061186692646
-tm_pt_14 -0.32923261148678096
-tm_pt_15 0.5524863522177359
-tm_pt_16 0.23451595442127693
-tm_glue_0 1
-WordPenalty -3.6942747832593694
-OOVPenalty -100.0