You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/07/29 17:22:52 UTC
[1/3] incubator-joshua git commit: JOSHUA-287 Refactored unit tests
to all use TestNG, removed no longer applicable tests,
refactored KenLM in order to make handling upstream errors easier
Repository: incubator-joshua
Updated Branches:
refs/heads/master a169f716d -> aadb4d687
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java b/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
index a99338a..8a68ab7 100644
--- a/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
+++ b/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
@@ -20,7 +20,7 @@
import static com.google.common.base.Charsets.UTF_8;
import static java.nio.file.Files.readAllBytes;
-import static org.junit.Assert.assertEquals;
+import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.nio.file.Path;
@@ -30,16 +30,15 @@ import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.apache.joshua.util.io.KenLmTestUtil;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
/**
* Reimplements the constrained phrase decoding test
*/
-@Ignore("re-enable as soon as kenlm native library support will be in place")
+
public class ConstrainedPhraseDecodingTest {
private static final String CONFIG = "resources/phrase_decoder/constrained.config";
@@ -49,20 +48,20 @@ public class ConstrainedPhraseDecodingTest {
private JoshuaConfiguration joshuaConfig = null;
private Decoder decoder = null;
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.readConfigFile(CONFIG);
- decoder = new Decoder(joshuaConfig, "");
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig, ""));
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
decoder.cleanUp();
decoder = null;
}
- @Test
+ @Test(enabled = false)
public void givenInput_whenConstrainedPhraseDecoding_thenOutputIsAsExpected() throws IOException {
final String translation = decode(INPUT).toString();
final String gold = new String(readAllBytes(GOLD_PATH), UTF_8);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java b/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
index b5bd612..f2fc6a7 100644
--- a/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
+++ b/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
@@ -20,7 +20,7 @@
import static com.google.common.base.Charsets.UTF_8;
import static java.nio.file.Files.readAllBytes;
-import static org.junit.Assert.assertEquals;
+import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.nio.file.Path;
@@ -30,16 +30,14 @@ import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.apache.joshua.util.io.KenLmTestUtil;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
/**
* Reimplements the constrained phrase decoding test
*/
-@Ignore("re-enable as soon as kenlm native library support will be in place")
public class PhraseDecodingTest {
private static final String CONFIG = "resources/phrase_decoder/config";
@@ -49,20 +47,20 @@ public class PhraseDecodingTest {
private JoshuaConfiguration joshuaConfig = null;
private Decoder decoder = null;
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.readConfigFile(CONFIG);
- decoder = new Decoder(joshuaConfig, "");
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig, ""));
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
decoder.cleanUp();
decoder = null;
}
- @Test
+ @Test(enabled = false)
public void givenInput_whenPhraseDecoding_thenOutputIsAsExpected() throws IOException {
final String translation = decode(INPUT).toString();
final String gold = new String(readAllBytes(GOLD_PATH), UTF_8);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/lattice/ArcTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/lattice/ArcTest.java b/src/test/java/org/apache/joshua/lattice/ArcTest.java
index a26a593..1ad020c 100644
--- a/src/test/java/org/apache/joshua/lattice/ArcTest.java
+++ b/src/test/java/org/apache/joshua/lattice/ArcTest.java
@@ -18,8 +18,6 @@
*/
package org.apache.joshua.lattice;
-import org.apache.joshua.lattice.Arc;
-
import org.testng.Assert;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/AlignmentMapTest.java b/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
index a28edc7..97e4906 100644
--- a/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
+++ b/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
@@ -18,8 +18,6 @@
*/
package org.apache.joshua.system;
-import static org.junit.Assert.*;
-
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -28,9 +26,13 @@ import java.util.Map;
import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.ff.FeatureVector;
import org.apache.joshua.decoder.ff.tm.Rule;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.internal.junit.ArrayAsserts.assertArrayEquals;
-import org.junit.Before;
-import org.junit.Test;
public class AlignmentMapTest {
@@ -39,7 +41,7 @@ public class AlignmentMapTest {
private static Map<Integer, List<Integer>> expectedAlignmentMap = null;
private static final int[] expectedNonTerminalPositions = {2,5};
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
Vocabulary.clear();
int[] sourceRhs = {Vocabulary.id("A1"),Vocabulary.id("A2"),-1,Vocabulary.id("B"),Vocabulary.id("C"),-2};
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/system/KenLmTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/KenLmTest.java b/src/test/java/org/apache/joshua/system/KenLmTest.java
index 8f1acb5..40514cd 100644
--- a/src/test/java/org/apache/joshua/system/KenLmTest.java
+++ b/src/test/java/org/apache/joshua/system/KenLmTest.java
@@ -16,34 +16,38 @@
* specific language governing permissions and limitations
* under the License.
*/
- package org.apache.joshua.system;
+package org.apache.joshua.system;
-import static org.apache.joshua.corpus.Vocabulary.registerLanguageModel;
-import static org.apache.joshua.corpus.Vocabulary.unregisterLanguageModels;
-import static org.junit.Assert.*;
import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.ff.lm.KenLM;
+import org.apache.joshua.util.io.KenLmTestUtil;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import static org.apache.joshua.corpus.Vocabulary.registerLanguageModel;
+import static org.apache.joshua.corpus.Vocabulary.unregisterLanguageModels;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
/**
* KenLM JNI interface tests.
* Loads libken.{so,dylib}.
- * If run in Eclipse, add -Djava.library.path=build/lib to JVM arguments
+ * If run in Eclipse, add -Djava.library.path=./lib to JVM arguments
* of the run configuration.
*/
-@Ignore("re-enable as soon as kenlm native library support will be in place")
+
public class KenLmTest {
private static final String LANGUAGE_MODEL_PATH = "resources/kenlm/oilers.kenlm";
+ private KenLM kenLm;
@Test
public void givenKenLm_whenQueryingForNgramProbability_thenProbIsCorrect() {
// GIVEN
- KenLM kenLm = new KenLM(3, LANGUAGE_MODEL_PATH);
+ KenLmTestUtil.Guard(() -> kenLm = new KenLM(3, LANGUAGE_MODEL_PATH));
+
int[] words = Vocabulary.addAll("Wayne Gretzky");
registerLanguageModel(kenLm);
@@ -52,13 +56,14 @@ public class KenLmTest {
// THEN
assertEquals("Found the wrong probability for 2-gram \"Wayne Gretzky\"", -0.99f, probability,
- Float.MIN_VALUE);
+ Float.MIN_VALUE);
}
-
+
@Test
public void givenKenLm_whenQueryingForNgramProbability_thenIdAndStringMethodsReturnTheSame() {
// GIVEN
- KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
+ KenLmTestUtil.Guard(() -> kenLm = new KenLM(LANGUAGE_MODEL_PATH));
+
registerLanguageModel(kenLm);
String sentence = "Wayne Gretzky";
String[] words = sentence.split("\\s+");
@@ -76,30 +81,18 @@ public class KenLmTest {
@Test
public void givenKenLm_whenIsKnownWord_thenReturnValuesAreCorrect() {
- KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
+ KenLmTestUtil.Guard(() -> kenLm = new KenLM(LANGUAGE_MODEL_PATH));
assertTrue(kenLm.isKnownWord("Wayne"));
assertFalse(kenLm.isKnownWord("Wayne2222"));
}
-
- @Test
- public void givenKenLm_whenIsLmOov_thenResultsAreCorrect() {
- KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
- registerLanguageModel(kenLm);
- int id = Vocabulary.id("Wayne");
- int id2 = Vocabulary.id("wekjfhaew;jvnae;goiawehco;eiwnf;oi");
- assertTrue(kenLm.isKnownWord(Vocabulary.word(id)));
- assertFalse(kenLm.isKnownWord(Vocabulary.word(id2)));
- assertFalse(kenLm.isOov(id));
- assertTrue(kenLm.isOov(id2));
- }
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
Vocabulary.clear();
unregisterLanguageModels();
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
Vocabulary.clear();
unregisterLanguageModels();
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
index 3901f40..092dbc6 100644
--- a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
+++ b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
@@ -16,9 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
- package org.apache.joshua.system;
-
-import static org.junit.Assert.assertTrue;
+package org.apache.joshua.system;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -33,10 +31,11 @@ import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.Translations;
import org.apache.joshua.decoder.io.TranslationRequestStream;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import static org.testng.Assert.assertTrue;
/**
* Integration test for multithreaded Joshua decoder tests. Grammar used is a
@@ -52,7 +51,7 @@ public class MultithreadedTranslationTests {
private int previousLogLevel;
private final static long NANO_SECONDS_PER_SECOND = 1_000_000_000;
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.search_algorithm = "cky";
@@ -89,7 +88,7 @@ public class MultithreadedTranslationTests {
Decoder.VERBOSE = 0;
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
this.decoder.cleanUp();
this.decoder = null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/StructuredOutputTest.java b/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
index f5e9d34..2c9e0bd 100644
--- a/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
+++ b/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
@@ -25,11 +25,10 @@ import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.Assert;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
/**
* Integration test for the complete Joshua decoder using a toy grammar that translates
@@ -54,7 +53,7 @@ public class StructuredOutputTest {
Arrays.asList(), Arrays.asList(7));
private static final double expectedScore = -17.0;
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.search_algorithm = "cky";
@@ -81,7 +80,7 @@ public class StructuredOutputTest {
// constructor/initialize)
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
decoder.cleanUp();
decoder = null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java b/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
index 6718858..4977d08 100644
--- a/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
+++ b/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
@@ -19,8 +19,8 @@
package org.apache.joshua.system;
import static java.util.Arrays.asList;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
import java.util.HashMap;
import java.util.List;
@@ -31,10 +31,9 @@ import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.StructuredTranslation;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
/**
* Integration test for the complete Joshua decoder using a toy grammar that translates
@@ -73,7 +72,7 @@ public class StructuredTranslationTest {
EXPECTED_FEATURES.put("OOVPenalty", 0.0f);
}
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.search_algorithm = "cky";
@@ -100,7 +99,7 @@ public class StructuredTranslationTest {
// constructor/initialize)
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
decoder.cleanUp();
decoder = null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/util/FormatUtilsTest.java b/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
index 84b418b..c5364dc 100644
--- a/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
+++ b/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
@@ -18,15 +18,17 @@
*/
package org.apache.joshua.util;
+import org.testng.annotations.Test;
+
import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
import static org.apache.joshua.util.FormatUtils.escapeSpecialSymbols;
import static org.apache.joshua.util.FormatUtils.isNonterminal;
import static org.apache.joshua.util.FormatUtils.ensureNonTerminalBrackets;
import static org.apache.joshua.util.FormatUtils.stripNonTerminalIndex;
import static org.apache.joshua.util.FormatUtils.unescapeSpecialSymbols;
-import static org.junit.Assert.*;
-
-import org.junit.Test;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
public class FormatUtilsTest {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/util/io/KenLmTestUtil.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/util/io/KenLmTestUtil.java b/src/test/java/org/apache/joshua/util/io/KenLmTestUtil.java
new file mode 100644
index 0000000..3fa7f6e
--- /dev/null
+++ b/src/test/java/org/apache/joshua/util/io/KenLmTestUtil.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import org.apache.joshua.decoder.ff.lm.KenLM;
+import org.testng.SkipException;
+
+/**
+ * Created by kellens on 7/12/16.
+ */
+public class KenLmTestUtil {
+
+ public static final int STACK_INSPECTION_LIMIT = 20;
+
+ // Catch any exceptions that could be thrown by KenLM, and skip the test if found.
+ public static void Guard(Runnable kenLmLoadingCode) {
+ try {
+ kenLmLoadingCode.run();
+ } catch (KenLM.KenLMLoadException e) {
+ skip();
+ }
+ catch (RuntimeException re) {
+ Throwable cause = re.getCause();
+ int stackCount = 0;
+
+ while (cause != null && stackCount < STACK_INSPECTION_LIMIT) {
+ stackCount++;
+ if (cause instanceof KenLM.KenLMLoadException) {
+ skip();
+ return;
+ } else {
+ cause = cause.getCause();
+ }
+ }
+
+ throw re;
+ }
+ }
+
+ private static void skip() {
+ throw new SkipException("Skipping test because KenLM.so/dylib was not found");
+ }
+}
[2/3] incubator-joshua git commit: JOSHUA-287 Refactored unit tests
to all use TestNG, removed no longer applicable tests,
refactored KenLM in order to make handling upstream errors easier
Posted by mj...@apache.org.
JOSHUA-287 Refactored unit tests to all use TestNG, removed no longer applicable tests, refactored KenLM in order to make handling upstream errors easier
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/94e863b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/94e863b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/94e863b8
Branch: refs/heads/master
Commit: 94e863b810f5caebdaf92f89f1fe723d6f58dc9c
Parents: a169f71
Author: Kellen Sunderland <ke...@amazon.com>
Authored: Tue Jul 12 19:34:52 2016 +0200
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Fri Jul 29 09:43:21 2016 +0200
----------------------------------------------------------------------
pom.xml | 15 +-
resources/kenlm/oilers.kenlm | Bin 0 -> 49011 bytes
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 53 +++---
.../ff/lm/StateMinimizingLanguageModel.java | 6 +-
.../apache/joshua/corpus/CorpusArrayTest.java | 177 ------------------
.../java/org/apache/joshua/corpus/SpanTest.java | 2 -
.../apache/joshua/corpus/VocabularyTest.java | 60 +++---
.../joshua/corpus/vocab/VocabularyTest.java | 180 ------------------
.../ArtificialGrammarAndCorpusCreater.java | 130 -------------
.../joshua/decoder/DecoderThreadTest.java | 172 -----------------
.../joshua/decoder/JoshuaDecoderTest.java | 83 ---------
.../joshua/decoder/TestConfigFileCreater.java | 184 -------------------
.../apache/joshua/decoder/TranslationsTest.java | 87 ---------
.../decoder/ff/ArityPhrasePenaltyFFTest.java | 64 -------
.../joshua/decoder/ff/lm/ArpaFileTest.java | 66 +------
.../decoder/ff/lm/LanguageModelFFTest.java | 33 ++--
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 44 ++---
.../class_lm/ClassBasedLanguageModelTest.java | 9 +-
.../io/TranslationRequestStreamTest.java | 158 ++++++++++++++++
.../decoder/io/TranslationRequestTest.java | 149 ---------------
.../kbest_extraction/KBestExtractionTest.java | 33 ++--
.../joshua/decoder/phrase/CoverageTest.java | 6 +-
.../ConstrainedPhraseDecodingTest.java | 21 +--
.../phrase/decode/PhraseDecodingTest.java | 20 +-
.../java/org/apache/joshua/lattice/ArcTest.java | 2 -
.../apache/joshua/system/AlignmentMapTest.java | 12 +-
.../org/apache/joshua/system/KenLmTest.java | 51 +++--
.../system/MultithreadedTranslationTests.java | 15 +-
.../joshua/system/StructuredOutputTest.java | 13 +-
.../system/StructuredTranslationTest.java | 15 +-
.../org/apache/joshua/util/FormatUtilsTest.java | 8 +-
.../apache/joshua/util/io/KenLmTestUtil.java | 59 ++++++
32 files changed, 420 insertions(+), 1507 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index da67edb..49d77fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -144,6 +144,10 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
+ <configuration>
+ <forkMode>once</forkMode>
+ <argLine>-Djava.library.path=${project.basedir}/lib</argLine>
+ </configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
@@ -235,18 +239,17 @@
<!-- Test Dependencies -->
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.9.10</version>
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.0.52-beta</version>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/resources/kenlm/oilers.kenlm
----------------------------------------------------------------------
diff --git a/resources/kenlm/oilers.kenlm b/resources/kenlm/oilers.kenlm
new file mode 100644
index 0000000..e343d7d
Binary files /dev/null and b/resources/kenlm/oilers.kenlm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java b/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
index 8fd9562..336189b 100644
--- a/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
@@ -19,7 +19,6 @@
package org.apache.joshua.decoder.ff.lm;
import org.apache.joshua.corpus.Vocabulary;
-import org.apache.joshua.decoder.ff.lm.NGramLanguageModel;
import org.apache.joshua.decoder.ff.state_maintenance.KenLMState;
import org.apache.joshua.util.FormatUtils;
import org.slf4j.Logger;
@@ -39,29 +38,12 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
private static final Logger LOG = LoggerFactory.getLogger(KenLM.class);
- static {
- try {
- System.loadLibrary("ken");
- } catch (UnsatisfiedLinkError e) {
- //TODO: send these prints to LOG.err
- LOG.error("* FATAL: Can't find libken.so (libken.dylib on OS X) in $JOSHUA/lib");
- LOG.error("* This probably means that the KenLM library didn't compile.");
- LOG.error("* Make sure that BOOST_ROOT is set to the root of your boost");
- LOG.error("* installation (it's not /opt/local/, the default), change to");
- LOG.error("* $JOSHUA, and type 'ant kenlm'. If problems persist, see the");
- LOG.error("* website (joshua.apache.org).");
- throw new RuntimeException(e);
- }
- }
-
private final long pointer;
// this is read from the config file, used to set maximum order
private final int ngramOrder;
// inferred from model file (may be larger than ngramOrder)
private final int N;
- // whether left-state minimization was requested
- private boolean minimizing;
private final static native long construct(String file_name);
@@ -85,13 +67,13 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
private final static native float probString(long ptr, int words[], int start);
- public final static native long createPool();
- public final static native void destroyPool(long pointer);
+ private final static native long createPool();
+
+ private final static native void destroyPool(long pointer);
public KenLM(int order, String file_name) {
+ pointer = initializeSystemLibrary(file_name);
ngramOrder = order;
-
- pointer = construct(file_name);
N = order(pointer);
}
@@ -101,11 +83,36 @@ public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
* @param file_name string path to an input file
*/
public KenLM(String file_name) {
- pointer = construct(file_name);
+ pointer = initializeSystemLibrary(file_name);
N = order(pointer);
ngramOrder = N;
}
+ private long initializeSystemLibrary(String file_name) {
+ try {
+ System.loadLibrary("ken");
+ return construct(file_name);
+ } catch (UnsatisfiedLinkError e) {
+ LOG.error("Can't find libken.so (libken.dylib on OS X) on the Java library path.");
+ throw new KenLMLoadException(e);
+ }
+ }
+
+ public class KenLMLoadException extends RuntimeException {
+
+ public KenLMLoadException(UnsatisfiedLinkError e) {
+ super(e);
+ }
+ }
+
+ public long createLMPool() {
+ return createPool();
+ }
+
+ public void destroyLMPool(long pointer) {
+ destroyPool(pointer);
+ }
+
public void destroy() {
destroy(pointer);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java b/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
index a7c9b16..4bec379 100644
--- a/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
@@ -43,7 +43,7 @@ import org.apache.joshua.decoder.segment_file.Sentence;
public class StateMinimizingLanguageModel extends LanguageModelFF {
// maps from sentence numbers to KenLM-side pools used to allocate state
- private static final ConcurrentHashMap<Integer, Long> poolMap = new ConcurrentHashMap<Integer, Long>();
+ private static final ConcurrentHashMap<Integer, Long> poolMap = new ConcurrentHashMap<>();
public StateMinimizingLanguageModel(FeatureVector weights, String[] args, JoshuaConfiguration config) {
super(weights, args, config);
@@ -118,7 +118,7 @@ public class StateMinimizingLanguageModel extends LanguageModelFF {
final int sentID = sentence.id();
// Since sentId is unique across threads, next operations are safe, but not atomic!
if (!poolMap.containsKey(sentID)) {
- poolMap.put(sentID, KenLM.createPool());
+ poolMap.put(sentID, ((KenLM) languageModel).createLMPool());
}
// Get the probability of applying the rule and the new state
@@ -170,7 +170,7 @@ public class StateMinimizingLanguageModel extends LanguageModelFF {
*/
public void destroyPool(int sentId) {
if (poolMap.containsKey(sentId))
- KenLM.destroyPool(poolMap.get(sentId));
+ ((KenLM) languageModel).destroyLMPool(poolMap.get(sentId));
poolMap.remove(sentId);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java b/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java
deleted file mode 100644
index 19cb20c..0000000
--- a/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.corpus;
-
-import java.util.logging.Logger;
-
-public class CorpusArrayTest {
-
- /** Logger for this class. */
- private static Logger logger =
- Logger.getLogger(CorpusArrayTest.class.getName());
-}
-
-// @Test
-// public void writePartsToDisk() {
-//
-// String filename = "data/tiny.en";
-// int numSentences = 5; // Should be 5 sentences in tiny.en
-// int numWords = 89; // Should be 89 words in tiny.en
-//
-//
-// try {
-//
-// // FIX: can't use createVocabulary(String) because we set numWords and numSentences
-// Vocabulary vocab = new Vocabulary();
-// SuffixArrayFactory.createVocabulary(filename, vocab);
-// Corpus corpus = SuffixArrayFactory.createCorpusArray(filename, vocab, numWords, numSentences);
-//
-// corpus.writeWordIDsToFile(filename+".bin");
-// corpus.writeSentenceLengthsToFile(filename+".sbin");
-//
-// MemoryMappedCorpusArray mmCorpus = new MemoryMappedCorpusArray(corpus.getVocabulary(), filename+".bin", numWords*4, filename+".sbin", numSentences*4);
-//
-// // For each word in the corpus,
-// for (int i=0; i<corpus.size(); i++) {
-//
-// // Verify that the memory-mapped corpus and the in-memory corpus have the same value
-// Assert.assertEquals(mmCorpus.getWordID(i), corpus.getWordID(i));
-// }
-//
-//
-// // For each sentence in the corpus
-// for (int i=0; i<corpus.sentences.length; i++) {
-//
-// // Verify that the sentence position in the memory-mapped corpus and the in-memory corpus have the same value
-// Assert.assertEquals(corpus.getSentencePosition(i), mmCorpus.getSentencePosition(i));
-// }
-//
-// } catch (IOException e) {
-// Assert.fail(e.getLocalizedMessage());
-// }
-//
-// }
-//
-// @Test
-// public void iterate() {
-//
-// String[] sentences = {
-// "scientists complete sequencing of the chromosome linked to early dementia",
-// "( afp , paris , january 2 ) an international team of scientists said that they have completed the sequencing of human chromosome 14 that is linked to many diseases , including the early-onset alzheimer's that may strike people in their 30s .",
-// "this is the fourth chromosome whose sequence has been completed to date . it comprises more than 87 million pairs of dna .",
-// "this study published in the weekly british scientific journal nature illustrates that the sequence of chromosome 14 comprises 1,050 genes and gene fragments .",
-// "the goal of geneticists is to provide diagnostic tools to identify defective genes that cause diseases so as to arrive eventually at treatments that can prevent those genes from malfunctioning ."
-// };
-//
-//
-//
-// // Tell System.out and System.err to use UTF8
-// FormatUtil.useUTF8();
-//
-// try {
-//
-// File sourceFile = File.createTempFile("source", new Date().toString());
-// PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");
-// for (String sentence : sentences) {
-// sourcePrintStream.println(sentence);
-// }
-// sourcePrintStream.close();
-// String corpusFileName = sourceFile.getAbsolutePath();
-//
-// Vocabulary vocabulary;
-//
-// logger.fine("Constructing vocabulary from file " + corpusFileName);
-// vocabulary = new Vocabulary();
-// int[] lengths = Vocabulary.initializeVocabulary(corpusFileName, vocabulary, true);
-//
-// logger.fine("Constructing corpus array from file " + corpusFileName);
-// Corpus corpus = SuffixArrayFactory.createCorpusArray(corpusFileName, vocabulary, lengths[0], lengths[1]);
-//
-// int expectedIndex = 0;
-// for (int actualIndex : corpus.corpusPositions()) {
-// Assert.assertEquals(actualIndex, expectedIndex);
-// expectedIndex += 1;
-// }
-//
-// Assert.assertEquals(corpus.size(), expectedIndex);
-//
-//
-// } catch (IOException e) {
-// Assert.fail("Unable to write temporary file. " + e.toString());
-// }
-//
-//
-//
-// }
-//
-//
-// @Test
-// public void writeAllToDisk() throws ClassNotFoundException {
-//
-// String filename = "data/tiny.en";
-// int numSentences = 5; // Should be 5 sentences in tiny.en
-// int numWords = 89; // Should be 89 words in tiny.en
-//
-//
-// try {
-//
-// // FIX: can't use createVocabulary(String) because we set numWords and numSentences
-// Vocabulary vocab = new Vocabulary();
-// Vocabulary.initializeVocabulary(filename, vocab, true);
-// CorpusArray corpus = SuffixArrayFactory.createCorpusArray(filename, vocab, numWords, numSentences);
-//
-// corpus.write(filename+".corpus", filename+".vocab", "UTF-8");
-//
-// MemoryMappedCorpusArray mmCorpus = new MemoryMappedCorpusArray(filename+".corpus", filename+".vocab");
-//
-// Assert.assertEquals(mmCorpus.size(), corpus.size());
-// Assert.assertEquals(mmCorpus.getNumSentences(), corpus.getNumSentences());
-//
-// // For each word in the corpus,
-// for (int i=0; i<corpus.size(); i++) {
-//
-// // Verify that the memory-mapped corpus and the in-memory corpus have the same value
-// Assert.assertEquals(mmCorpus.getWordID(i), corpus.getWordID(i));
-// }
-//
-//
-// // For each sentence in the corpus
-// for (int i=0; i<corpus.sentences.length; i++) {
-//
-// // Verify that the sentence start position in the memory-mapped corpus and the in-memory corpus have the same value
-// Assert.assertEquals(mmCorpus.getSentencePosition(i), corpus.getSentencePosition(i));
-//
-// // Verify that the sentence end position in the memory-mapped corpus and the in-memory corpus have the same value
-// Assert.assertEquals(mmCorpus.getSentenceEndPosition(i), corpus.getSentenceEndPosition(i));
-//
-// // Verify that the phrase corresponding to this sentence is the same
-// Phrase sentence = corpus.getSentence(i);
-// Phrase mmSentence = mmCorpus.getSentence(i);
-// Assert.assertNotNull(sentence);
-// Assert.assertNotNull(mmSentence);
-// Assert.assertEquals(mmSentence, sentence);
-// }
-//
-// } catch (IOException e) {
-// Assert.fail(e.getLocalizedMessage());
-// }
-//
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/corpus/SpanTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/corpus/SpanTest.java b/src/test/java/org/apache/joshua/corpus/SpanTest.java
index 3558b79..70fe67a 100644
--- a/src/test/java/org/apache/joshua/corpus/SpanTest.java
+++ b/src/test/java/org/apache/joshua/corpus/SpanTest.java
@@ -18,8 +18,6 @@
*/
package org.apache.joshua.corpus;
-import org.apache.joshua.corpus.Span;
-
import org.testng.Assert;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/corpus/VocabularyTest.java b/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
index 834d68b..e3042ed 100644
--- a/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
+++ b/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
@@ -18,36 +18,34 @@
*/
package org.apache.joshua.corpus;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
import static org.apache.joshua.util.FormatUtils.isNonterminal;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
import java.io.File;
import java.io.IOException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
public class VocabularyTest {
private static final String WORD1 = "word1";
private static final String WORD2 = "word2";
private static final String NON_TERMINAL = "[X]";
private static final String GOAL = "[GOAL]";
- @Before
+ @BeforeMethod
public void init() {
Vocabulary.clear();
}
-
- @After
+
+ @AfterMethod
public void deinit() {
Vocabulary.clear();
}
-
+
@Test
public void givenVocabulary_whenEmpty_thenOnlyContainsUnknownWord() {
assertTrue(Vocabulary.hasId(Vocabulary.UNKNOWN_ID));
@@ -56,7 +54,7 @@ public class VocabularyTest {
assertEquals(Vocabulary.UNKNOWN_WORD, Vocabulary.word(Vocabulary.UNKNOWN_ID));
assertEquals(1, Vocabulary.size());
}
-
+
@Test
public void givenVocabulary_whenNewWord_thenMappingIsAdded() {
final int FIRST_WORD_ID = 1;
@@ -68,7 +66,7 @@ public class VocabularyTest {
assertEquals(WORD1, Vocabulary.word(FIRST_WORD_ID));
assertEquals(2, Vocabulary.size());
}
-
+
@Test
public void givenVocabulary_whenCheckingStringInBracketsOrNegativeNumber_thenIsNonTerminal() {
//non-terminals
@@ -79,51 +77,51 @@ public class VocabularyTest {
assertFalse(isNonterminal("["));
assertFalse(isNonterminal("]"));
assertFalse(isNonterminal(""));
-
+
//negative numbers indicate non-terminals
assertTrue(isNonterminal(-1));
assertTrue(isNonterminal(-5));
-
+
//positive numbers indicate terminals:
assertFalse(isNonterminal(0));
assertFalse(isNonterminal(5));
+
+
}
-
+
@Test
public void givenVocabulary_whenNonTerminal_thenReturnsStrictlyPositiveNonTerminalIndices() {
final int FIRST_NON_TERMINAL_INDEX = 1;
assertTrue(Vocabulary.id(NON_TERMINAL) < 0);
assertTrue(Vocabulary.hasId(FIRST_NON_TERMINAL_INDEX));
assertTrue(Vocabulary.hasId(-FIRST_NON_TERMINAL_INDEX));
-
+
assertTrue(Vocabulary.id("") > 0);
assertTrue(Vocabulary.id(WORD1) > 0);
-
+
final int SECOND_NON_TERMINAL_INDEX = 4;
assertTrue(Vocabulary.id(GOAL) < 0);
assertTrue(Vocabulary.hasId(SECOND_NON_TERMINAL_INDEX));
assertTrue(Vocabulary.hasId(-SECOND_NON_TERMINAL_INDEX));
-
+
assertTrue(Vocabulary.id(WORD2) > 0);
}
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
+
@Test
public void givenVocabulary_whenWritenAndReading_thenVocabularyStaysTheSame() throws IOException {
- File vocabFile = folder.newFile();
-
+ File vocabFile = File.createTempFile( "vocab", "tmp");
+ vocabFile.deleteOnExit();
+
int id1 = Vocabulary.id(WORD1);
int id2 = Vocabulary.id(NON_TERMINAL);
int id3 = Vocabulary.id(WORD2);
-
+
Vocabulary.write(vocabFile.getAbsolutePath());
-
+
Vocabulary.clear();
-
+
Vocabulary.read(vocabFile);
-
+
assertEquals(4, Vocabulary.size()); //unknown word + 3 other words
assertTrue(Vocabulary.hasId(id1));
assertTrue(Vocabulary.hasId(id2));
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java b/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java
deleted file mode 100644
index c1af5ab..0000000
--- a/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.corpus.vocab;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Date;
-import org.apache.joshua.corpus.Vocabulary;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- *
- *
- * @author Lane Schwartz
- */
-public class VocabularyTest {
-
- /** [X], [X,1], [X,2], [S], [S,1] <unk>, <s>, </s>, -pau-*/
- int numBuiltInSymbols = 9;
-
- /** <unk>, <s>, </s>, -pau- */
- int numBuiltInTerminals = 4;
-
- @Test
- public void basicVocabTest() {
-
- Vocabulary vocab1 = new Vocabulary();
- Vocabulary vocab2 = new Vocabulary();
-
- Assert.assertEquals(vocab1, vocab2);
-
- Assert.assertFalse(vocab1.size() == 0);
- //Assert.assertTrue(vocab1.intToString.get(0)==Vocabulary.UNKNOWN_WORD_STRING);
- //Assert.assertFalse(vocab1.getWords().isEmpty());
- // Assert.assertTrue(vocab1.getWords(0)==Vocabulary.UNKNOWN_WORD_STRING);
- // Assert.assertEquals(vocab1.getWords(), vocab1.intToString.values());
-
- Assert.assertNotEquals(vocab1.size(), numBuiltInSymbols);
- // Assert.assertEquals(vocab1.getWord(Vocabulary.UNKNOWN_WORD), Vocabulary.UNKNOWN_WORD_STRING);
-
- //Assert.assertEquals(vocab1.getID("sample"), Vocabulary.UNKNOWN_WORD);
- //Assert.assertEquals(vocab1.getID(null), Vocabulary.UNKNOWN_WORD);
-
- // Assert.assertFalse(vocab1.terminalToInt.isEmpty());
- // Assert.assertEquals(vocab1.terminalToInt.size(), this.numBuiltInTerminals);
- // Assert.assertFalse(vocab1.isFixed);
- //
- // vocab1.fixVocabulary();
- // Assert.assertTrue(vocab1.isFixed);
-
- // Assert.assertEquals(vocab1.getID(Vocabulary.X_STRING), -1);
- // Assert.assertEquals(vocab1.getID(Vocabulary.X1_STRING), -2);
- // Assert.assertEquals(vocab1.getID(Vocabulary.X2_STRING), -3);
- //
- // Assert.assertEquals(vocab1.getWord(-1), Vocabulary.X_STRING);
- // Assert.assertEquals(vocab1.getWord(-2), Vocabulary.X1_STRING);
- // Assert.assertEquals(vocab1.getWord(-3), Vocabulary.X2_STRING);
-
-
-
- // Assert.assertFalse(vocab2.intToString.isEmpty());
- // Assert.assertTrue(vocab2.intToString.get(0)==Vocabulary.UNKNOWN_WORD_STRING);
- // Assert.assertFalse(vocab2.getWords().isEmpty());
- // Assert.assertTrue(vocab2.getWord(0)==Vocabulary.UNKNOWN_WORD_STRING);
- // Assert.assertEquals(vocab2.getWords(), vocab2.intToString.values());
-
- Assert.assertNotEquals(vocab2.size(), numBuiltInSymbols);
- // Assert.assertEquals(vocab2.getWord(Vocabulary.UNKNOWN_WORD), Vocabulary.UNKNOWN_WORD_STRING);
-
- // Assert.assertEquals(vocab2.getID("sample"), Vocabulary.UNKNOWN_WORD);
- // Assert.assertEquals(vocab2.getID(null), Vocabulary.UNKNOWN_WORD);
-
- // Assert.assertFalse(vocab2.terminalToInt.isEmpty());
- // Assert.assertEquals(vocab2.terminalToInt.size(), this.numBuiltInTerminals);
- // Assert.assertTrue(vocab2.isFixed);
- }
-
- @Test
- public void verifyWordIDs() throws IOException {
-
- // Adam Lopez's example...
- String corpusString = "it makes him and it mars him , it sets him on and it takes him off .";
- // String queryString = "it persuades him and it disheartens him";
-
- String sourceFileName;
- {
- File sourceFile = File.createTempFile("source", new Date().toString());
- PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");
- sourcePrintStream.println(corpusString);
- sourcePrintStream.close();
- sourceFileName = sourceFile.getAbsolutePath();
- }
-
- Vocabulary vocab = new Vocabulary();
- // Vocabulary.initializeVocabulary(sourceFileName, vocab, true);
-
-// Assert.assertEquals(vocab.getWords(Vocabulary.id("it")), "it");
-// Assert.assertEquals(vocab.getWord(vocab.getID("makes")), "makes");
-// Assert.assertEquals(vocab.getWord(vocab.getID("him")), "him");
-// Assert.assertEquals(vocab.getWord(vocab.getID("and")), "and");
-// Assert.assertEquals(vocab.getWord(vocab.getID("mars")), "mars");
-// Assert.assertEquals(vocab.getWord(vocab.getID(",")), ",");
-// Assert.assertEquals(vocab.getWord(vocab.getID("sets")), "sets");
-// Assert.assertEquals(vocab.getWord(vocab.getID("on")), "on");
-// Assert.assertEquals(vocab.getWord(vocab.getID("takes")), "takes");
-// Assert.assertEquals(vocab.getWord(vocab.getID("off")), "off");
-
- // Assert.assertEquals(vocab.getWord(vocab.getID("persuades")), Vocabulary.UNKNOWN_WORD_STRING);
- // Assert.assertEquals(vocab.getWord(vocab.getID("disheartens")), Vocabulary.UNKNOWN_WORD_STRING);
- }
-
- @SuppressWarnings("static-access")
- @Test(enabled=false)
- public void loadVocabFromFile() {
-
- String filename = "data/tiny.en";
- int numSentences = 5; // Should be 5 sentences in tiny.en
- int numWords = 89; // Should be 89 words in tiny.en
- int numUniqWords = 60; // Should be 60 unique words in tiny.en
-
- Vocabulary vocab = new Vocabulary();
- Vocabulary vocab2 = new Vocabulary();
-
- Assert.assertTrue(vocab.equals(vocab2));
- Assert.assertTrue(vocab2.equals(vocab));
- Assert.assertEquals(vocab, vocab2);
-
- try {
- vocab.read(new File(getClass().getClassLoader().getResource(filename).getFile()));
- //int[] result = Vocabulary.initializeVocabulary(filename, vocab, true);
- Assert.assertNotNull(vocab);
- Assert.assertEquals(vocab.size(), 2);
- //Assert.assertEquals(vocab.getWords(numWords), numWords);
- // Assert.assertEquals(result[1], numSentences);
-
- //Assert.assertTrue(vocab.isFixed);
- Assert.assertEquals(Vocabulary.size(), numUniqWords+numBuiltInSymbols);
-
- } catch (IOException e) {
- Assert.fail("Error processing " + filename +"; Reason: " + e);
- }
-
- Assert.assertFalse(vocab.equals(vocab2));
-
- try {
- vocab2.read(new File(filename));
- //int[] result = Vocabulary.initializeVocabulary(filename, vocab2, true);
- Assert.assertNotNull(vocab2);
- Assert.assertEquals(vocab2.size(), 2);
- // Assert.assertEquals(result[0], numWords);
- // Assert.assertEquals(result[1], numSentences);
-
- // Assert.assertTrue(vocab2.isFixed);
- Assert.assertEquals(Vocabulary.size(), numUniqWords+numBuiltInSymbols);
-
- } catch (IOException e) {
- Assert.fail("Could not load file " + filename);
- }
-
- Assert.assertEquals(vocab, vocab2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java b/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java
deleted file mode 100644
index 5cc5996..0000000
--- a/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.joshua.util.FileUtility;
-
-public class ArtificialGrammarAndCorpusCreater {
-
- private static final String JOSHUA_RULE_SEPARATOR = " ||| ";
- private static final String ARTIFICAL_TERMINAL_RULE1 = "[T1]" + JOSHUA_RULE_SEPARATOR + "garcon"
- + JOSHUA_RULE_SEPARATOR + "boy" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_TERMINAL_RULE2 = "[T2]" + JOSHUA_RULE_SEPARATOR + "fille"
- + JOSHUA_RULE_SEPARATOR + "girl" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_TERMINAL_RULE3 = "[T3]" + JOSHUA_RULE_SEPARATOR + "garcon"
- + JOSHUA_RULE_SEPARATOR + "mister" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_TERMINAL_RULE4 = "[T4]" + JOSHUA_RULE_SEPARATOR + "fille"
- + JOSHUA_RULE_SEPARATOR + "woman" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_TERMINAL_RULE5 = "[T5]" + JOSHUA_RULE_SEPARATOR + "fille"
- + JOSHUA_RULE_SEPARATOR + "lady" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_NONTERTERMINAL_RULE1 = "[NT1]" + JOSHUA_RULE_SEPARATOR
- + "le [T1,1] aime la [T2,2]" + JOSHUA_RULE_SEPARATOR + "the [T1,1] loves the [T2,2]"
- + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_NONTERTERMINAL_RULE_INVERTED = "[NT1]"
- + JOSHUA_RULE_SEPARATOR + "le [T1,1] aime la [T2,2]" + JOSHUA_RULE_SEPARATOR
- + "the [T2,2] loves the [T1,1]" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
- private static final String ARTIFICAL_TERMINAL_RULE6 = "[T6]" + JOSHUA_RULE_SEPARATOR + "garcon"
- + JOSHUA_RULE_SEPARATOR + "sir" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
-
- private static final String GLUE_RULE_BEGIN = "[GOAL] ||| <s> ||| <s> ||| 0";
- private static final String GLUE_RULE_NT = "[GOAL] ||| [GOAL,1] [NT1,2] ||| [GOAL,1] [NT1,2] ||| -1";
- private static final String GLUE_RULE_END = "[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0";
-
- private static final String TEST_SENTENCE1 = "le garcon aime la fille";
-
- private static final List<String> getArtificalGrammarsList1() {
- List<String> result = Arrays.asList(ARTIFICAL_TERMINAL_RULE1, ARTIFICAL_TERMINAL_RULE2,
- ARTIFICAL_TERMINAL_RULE3, ARTIFICAL_TERMINAL_RULE4, ARTIFICAL_TERMINAL_RULE5,
- ARTIFICAL_TERMINAL_RULE6, ARTIFICAL_NONTERTERMINAL_RULE1);
- return result;
- }
-
- private static List<String> getArtificalGrammarsList2() {
- List<String> result = new ArrayList<String>(getArtificalGrammarsList1());
- result.add(ARTIFICAL_NONTERTERMINAL_RULE_INVERTED);
- return result;
- }
-
- private static final List<String> ARTIFICIAL_GLUE_GRAMMAR_RULES_LIST = Arrays.asList(
- GLUE_RULE_BEGIN, GLUE_RULE_NT, GLUE_RULE_END);
-
- private final String mainGrammarFilePath;
- private final String glueGrammarFilePath;
- private final String testSentencesFilePath;
-
- private ArtificialGrammarAndCorpusCreater(String mainGrammarFilePath, String glueGrammarFilePath,
- String testSentencesFilePath) {
- this.mainGrammarFilePath = mainGrammarFilePath;
- this.glueGrammarFilePath = glueGrammarFilePath;
- this.testSentencesFilePath = testSentencesFilePath;
- }
-
- public static ArtificialGrammarAndCorpusCreater createArtificialGrammarAndCorpusCreater(
- String mainGrammarFilePath, String glueGrammarFilePath, String testSentencesFilePath) {
- return new ArtificialGrammarAndCorpusCreater(mainGrammarFilePath, glueGrammarFilePath,
- testSentencesFilePath);
- }
-
- private static final void writeFile(String filePath, List<String> lines) {
- BufferedWriter outputWriter = null;
- try {
- outputWriter = new BufferedWriter(new FileWriter(filePath));
- for (int i = 0; i < lines.size() - 1; i++) {
- outputWriter.write(lines.get(i) + "\n");
- }
- if (!lines.isEmpty()) {
- outputWriter.write(lines.get(lines.size() - 1));
- }
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } finally {
- FileUtility.closeCloseableIfNotNull(outputWriter);
- }
- }
-
- protected final void writeMainGrammar(boolean includeInvertingNonterminalRule) {
- List<String> ruleList;
- if(includeInvertingNonterminalRule)
- {
- ruleList = getArtificalGrammarsList2();
- }
- else{
- ruleList = getArtificalGrammarsList1();
- }
-
- writeFile(mainGrammarFilePath,ruleList);
- }
-
- protected final void writeGlueGrammar() {
- writeFile(glueGrammarFilePath, ARTIFICIAL_GLUE_GRAMMAR_RULES_LIST);
- }
-
- protected final void writeTestSentencesFile1() {
- writeFile(testSentencesFilePath, Arrays.asList(TEST_SENTENCE1));
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java b/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java
deleted file mode 100644
index 326ab23..0000000
--- a/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Date;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- * Unit tests for decoder thread.
- *
- * @author Lane Schwartz
- * @version $LastChangedDate$
- */
-public class DecoderThreadTest {
-
- @Test
- public void setup() {
-
- String[] sourceSentences = {
- "a b c d",
- "a b c d",
- "a b c d"
- };
-
- String[] targetSentences = {
- "w x y z",
- "w t u v",
- "s x y z"
- };
-
- String[] alignmentLines = {
- "0-0 1-1 2-2 3-3",
- "0-0 1-1 2-2 3-3",
- "0-0 1-1 2-2 3-3"
- };
-
- String[] testSentences = {
- "a b c"
- };
-
- try {
-
- // Set up source corpus
- File sourceFile = File.createTempFile("source", new Date().toString());
- PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");
- for (String sentence : sourceSentences) {
- sourcePrintStream.println(sentence);
- }
- sourcePrintStream.close();
- String sourceCorpusFileName = sourceFile.getAbsolutePath();
-
-// Vocabulary vocabulary = new Vocabulary();
-// int[] sourceLengths = Vocabulary.initializeVocabulary(sourceCorpusFileName, vocabulary, true);
-// Assert.assertEquals(sourceLengths.length, 2);
-// int numberOfSentences = sourceLengths[1];
-//
-// Corpus sourceCorpus = SuffixArrayFactory.createCorpusArray(sourceCorpusFileName, vocabulary, sourceLengths[0], sourceLengths[1]);
-
-
- // Set up target corpus
- File targetFile = File.createTempFile("target", new Date().toString());
- PrintStream targetPrintStream = new PrintStream(targetFile, "UTF-8");
- for (String sentence : targetSentences) {
- targetPrintStream.println(sentence);
- }
- targetPrintStream.close();
- String targetCorpusFileName = targetFile.getAbsolutePath();
-
-// int[] targetLengths = Vocabulary.initializeVocabulary(targetCorpusFileName, vocabulary, true);
-// Assert.assertEquals(targetLengths.length, sourceLengths.length);
-// for (int i=0, n=targetLengths.length; i<n; i++) {
-// Assert.assertEquals(targetLengths[i], sourceLengths[i]);
-// }
-//
-// Corpus targetCorpus = SuffixArrayFactory.createCorpusArray(targetCorpusFileName, vocabulary, targetLengths[0], targetLengths[1]);
-
-
- // Construct alignments data structure
- File alignmentsFile = File.createTempFile("alignments", new Date().toString());
- PrintStream alignmentsPrintStream = new PrintStream(alignmentsFile, "UTF-8");
- for (String sentence : alignmentLines) {
- alignmentsPrintStream.println(sentence);
- }
- alignmentsPrintStream.close();
- String alignmentFileName = alignmentsFile.getAbsolutePath();
-
-// AlignmentGrids grids = new AlignmentGrids(
-// new Scanner(alignmentsFile),
-// sourceCorpus,
-// targetCorpus,
-// numberOfSentences);
-
-
- // Set up test corpus
- File testFile = File.createTempFile("test", new Date().toString());
- PrintStream testPrintStream = new PrintStream(testFile, "UTF-8");
- for (String sentence : testSentences) {
- testPrintStream.println(sentence);
- }
- testPrintStream.close();
- String testFileName = testFile.getAbsolutePath();
-
- // Filename of the extracted rules file.
- String rulesFileName; {
- File rulesFile = File.createTempFile("rules", new Date().toString());
- rulesFileName = rulesFile.getAbsolutePath();
- }
-
- String joshDirName; {
- File joshDir = File.createTempFile(new Date().toString(), "josh");
- joshDirName = joshDir.getAbsolutePath();
- joshDir.delete();
- }
-
-
-// Compile compileJoshDir = new Compile();
-// compileJoshDir.setSourceCorpus(sourceCorpusFileName);
-// compileJoshDir.setTargetCorpus(targetCorpusFileName);
-// compileJoshDir.setAlignments(alignmentFileName);
-// compileJoshDir.setOutputDir(joshDirName);
-// compileJoshDir.execute();
-//
-// ExtractRules extractRules = new ExtractRules();
-// extractRules.setJoshDir(joshDirName);
-// extractRules.setTestFile(testFileName);
-// extractRules.setOutputFile(rulesFileName);
-// extractRules.execute();
-
- } catch (IOException e) {
- Assert.fail("Unable to write temporary file. " + e.toString());
- }
-// } catch (ClassNotFoundException e) {
-// Assert.fail("Unable to extract rules. " + e.toString());
-// }
- }
-
- @Test
- public void basicSuffixArrayGrammar() {
-
- // Write configuration to temp file on disk
- // String configFile;
-
-
- // JoshuaDecoder decoder =
- // JoshuaDecoder.getUninitalizedDecoder(configFile);
-
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java b/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java
deleted file mode 100644
index 2a878f3..0000000
--- a/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Scanner;
-
-import org.testng.Assert;
-import org.testng.annotations.Parameters;
-import org.testng.annotations.Test;
-
-/**
- * Performs regression tests to verify that the decoder produces expected output
- * on known data sets.
- *
- * @author Lane Schwartz
- */
-public class JoshuaDecoderTest {
-
- @Parameters({ "configFile", "sourceInput", "referenceOutput" })
- @Test
- public void regressionTest(String configFile, String sourceInput, String referenceOutput)
- throws IOException {
-
- File referenceFile = new File(referenceOutput);
- File output = File.createTempFile("output", null);// ,
- // referenceFile.getParentFile());
-
- String[] args = { configFile, sourceInput, output.getAbsoluteFile().toString() };
- JoshuaDecoder.main(args);
-
- Scanner resultScanner = new Scanner(output);
- Scanner refScanner = new Scanner(referenceFile);
-
- while (resultScanner.hasNextLine() && refScanner.hasNextLine()) {
-
- String resultLine = resultScanner.nextLine();
- String refLine = refScanner.nextLine();
-
- String[] resultParts = resultLine.split(" \\|\\|\\| ");
- String[] refParts = refLine.split(" \\|\\|\\| ");
-
- Assert.assertEquals(resultParts.length, 4);
- Assert.assertEquals(refParts.length, 4);
-
- Assert.assertEquals(Integer.parseInt(resultParts[0]), Integer.parseInt(refParts[0]));
- Assert.assertEquals(resultParts[1], refParts[1]);
-
- String[] resultFeatures = resultParts[2].split(" ");
- String[] refFeatures = refParts[2].split(" ");
-
- Assert.assertEquals(resultFeatures.length, 5);
- Assert.assertEquals(refFeatures.length, 5);
-
- float acceptableDelta = 0.001f;
- for (int i = 0; i < refFeatures.length; i++) {
- Assert.assertEquals(Float.valueOf(resultFeatures[i]), Float.valueOf(refFeatures[i]),
- acceptableDelta);
- }
- }
-
- resultScanner.close();
- refScanner.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java b/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java
deleted file mode 100644
index 5399bab..0000000
--- a/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.List;
-import org.apache.joshua.util.FileUtility;
-
-public class TestConfigFileCreater {
-
-
- protected static String LANGUAGE_MODEL_FILE_NAME = "lm.gz";
- private static final String NL = "\n";
- private static final Double NEW_FEATURES_WEIGHT = 0.2;
-
- private final String testTempFilesFolderName;
- private final String mainGrammarFileName;
- private final String glueGrammarFileName;
- private final List<Double> phraseTableWeights;
- private final boolean useSoftSyntacticDecoding;
- private final boolean switchOfPruning;
-
- private TestConfigFileCreater(String testTemFilesFolderName, String mainGrammarFileName,
- String glueGrammarFileName, List<Double> phraseTableWeights,
- boolean useSoftSyntacticDecoding, boolean switchOfPruning) {
- this.testTempFilesFolderName = testTemFilesFolderName;
- this.mainGrammarFileName = mainGrammarFileName;
- this.glueGrammarFileName = glueGrammarFileName;
- this.phraseTableWeights = phraseTableWeights;
- this.useSoftSyntacticDecoding = useSoftSyntacticDecoding;
- this.switchOfPruning = switchOfPruning;
- }
-
- public static TestConfigFileCreater createFeaturesTestConfigFileCreater(
- String testTemFilesFolderName, String mainGrammarFileName, String glueGrammarFileName,
-
- List<Double> phraseTableWeights, boolean useSoftSyntacticDecoding, boolean switchOfPruning) {
- return new TestConfigFileCreater(testTemFilesFolderName, mainGrammarFileName,
- glueGrammarFileName, phraseTableWeights, useSoftSyntacticDecoding, switchOfPruning);
- }
-
- private final String createGlueGrammarFileSpecificationLine() {
- return "tm = thrax glue -1 " + "./" + testTempFilesFolderName + "/" + glueGrammarFileName;
- }
-
- private final String createMainGrammarFileSpecificationLine() {
- return "tm = thrax pt 12 " + "./" + testTempFilesFolderName + "/" + mainGrammarFileName;
- }
-
- private static String getFeatureSwitchOnString(String featureFunctionName) {
- return "feature-function = " + featureFunctionName;
- }
-
- public String getPruningSpecification() {
- if (switchOfPruning) {
- return "pop-limit = 0" + NL;
- } else {
- return "pop-limit = 100" + NL;
- }
- }
-
- // Large String containing the mostly static, partly dynamic generated mose config
- // file contents used for the test
- private final String getJoshuaConfigFileFirstPart(boolean useSoftSyntacticDecoding) {
- String result = "lm = kenlm 5 false false 100 " + createFullPath(LANGUAGE_MODEL_FILE_NAME) + NL
- + createMainGrammarFileSpecificationLine() + NL + createGlueGrammarFileSpecificationLine()
- + NL + "mark_oovs=false" + NL + "#tm config" + NL + "default_non_terminal = OOV" + NL
- + "goalSymbol = GOAL" + NL + "#pruning config" + NL + getPruningSpecification()
- + JoshuaConfiguration.SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME + " = "
- + useSoftSyntacticDecoding + NL + "#nbest config" + NL + "use_unique_nbest = true" + NL
-
- + "top_n = 100" // + NL +
- // "feature-function = OOVPenalty"
- + NL + "feature-function = WordPenalty";
- return result;
- }
-
- private final String createPhraseTableSpecificationString() {
- String result = "";
- for (int i = 0; i < phraseTableWeights.size(); i++) {
- double phraseTableWeight = phraseTableWeights.get(i);
- result += "tm_pt_" + i + " " + phraseTableWeight + NL;
- }
- return result;
- }
-
- private final String getMosesConfigFilePart2() {
- String retsult = "###### model weights" + NL + "#lm order weight" + NL
- + "WordPenalty -3.0476045270236662" + NL + createPhraseTableSpecificationString()
- + "lm_0 1.3200621467242506"
- // "#phrasemodel owner column(0-indexed)"
- + NL + "tm_glue_0 1" + NL + "oovpenalty -100.0" + NL;
- return retsult;
- }
-
- // private static final int NO_PHRASE_WEIGTHS = 22;
-
- /*
- * private static String createPhraseWeightsSpecification() { String result =
- * "#phrasemodel owner column(0-indexed) weight" + NL; for (int i = 0; i < NO_PHRASE_WEIGTHS; i++)
- * { result += "tm_pt_" + i + 0.5; } return result; }
- */
-
- private static String createFeatureWeightSpecifications(List<String> featureNames,
- double featureWeight) {
- String result = "";
- for (String featureName : featureNames) {
- result += featureName + " " + featureWeight + "\n";
- }
- return result;
- }
-
- protected String createJoshuaConfigFileContentsWithExtraFeatures(String featureFunctionName,
- List<String> featureNames) {
- String result = createJoshuaConfigFileContents(featureFunctionName);
- result += createFeatureWeightSpecifications(featureNames, NEW_FEATURES_WEIGHT);
- return result;
- }
-
- protected String createJoshuaConfigFileContents(String featureFunctionName) {
- String result = getJoshuaConfigFileFirstPart(useSoftSyntacticDecoding);
- result += NL + getFeatureSwitchOnString(featureFunctionName) + NL;
- result += getMosesConfigFilePart2();
- return result;
- }
-
- protected String createJoshuaConfigFileContents() {
- String result = getJoshuaConfigFileFirstPart(useSoftSyntacticDecoding);
- result += NL;
- result += getMosesConfigFilePart2();
- return result;
- }
-
- protected static void writeContents(String filePath, String contents) {
- BufferedWriter outputWriter = null;
- try {
- outputWriter = new BufferedWriter(new FileWriter(filePath));
- outputWriter.write(contents);
- } catch (IOException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- } finally {
- FileUtility.closeCloseableIfNotNull(outputWriter);
- }
- }
-
- String createFullPath(String fileName) {
- return testTempFilesFolderName + "/" + fileName;
- }
-
- protected void writeBasicJoshuaConfigFile(String configFileName) {
- writeContents(createFullPath(configFileName), createJoshuaConfigFileContents());
- }
-
- protected void writeBasicJoshuaConfigFile(String configFileName, String featureFunctionName) {
- writeContents(createFullPath(configFileName),
- createJoshuaConfigFileContents(featureFunctionName));
- }
-
- protected void writeJoshuaExtraFeaturesConfigFile(String configFileName,
- String featureFunctionName, List<String> featureNames) {
- TestConfigFileCreater.writeContents(createFullPath(configFileName),
- createJoshuaConfigFileContentsWithExtraFeatures(featureFunctionName, featureNames));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/TranslationsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/TranslationsTest.java b/src/test/java/org/apache/joshua/decoder/TranslationsTest.java
deleted file mode 100644
index 9d2cb34..0000000
--- a/src/test/java/org/apache/joshua/decoder/TranslationsTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import static org.testng.Assert.*;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-
-import org.testng.annotations.Test;
-import org.testng.annotations.BeforeTest;
-import org.apache.joshua.decoder.io.TranslationRequestStream;
-import org.testng.annotations.AfterTest;
-
-public class TranslationsTest {
- private final JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
- @BeforeTest
- public void beforeTest() {
- }
-
- @AfterTest
- public void afterTest() {
- }
-
-
- @Test(enabled = false)
- public void Translations() {
- throw new RuntimeException("Test not implemented");
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#next()}.
- */
- @Test(enabled = false)
- public void testNext() {
- fail("Not yet implemented");
- }
-
- @Test(enabled = false)
- public void iterator() {
- throw new RuntimeException("Test not implemented");
- }
-
- // @Test(expectedExceptions = TestException.class)
- @Test(enabled = false)
- public void next() {
- byte[] data = "1\n2\n".getBytes();
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequestStream request = new TranslationRequestStream(
- new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
- Translations translations = new Translations(request);
- assertEquals(translations.next().getSourceSentence().source(), "1");
- // Remove the next two.
- assertEquals(translations.next().getSourceSentence().source(), "2");
- // Should throw exception
- translations.next();
- translations.next();
- }
-
- @Test(enabled = false)
- public void record() {
- throw new RuntimeException("Test not implemented");
- }
-
- @Test(enabled = false)
- public void remove() {
- throw new RuntimeException("Test not implemented");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java b/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java
deleted file mode 100644
index 9899298..0000000
--- a/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-//package org.apache.joshua.decoder.ff;
-//
-//import org.apache.joshua.decoder.ff.tm.BilingualRule;
-//import org.apache.joshua.decoder.ff.tm.MonolingualRule;
-//import org.apache.joshua.decoder.ff.tm.Rule;
-//
-//import org.testng.Assert;
-//import org.testng.annotations.Test;
-//
-///**
-// * Unit tests for ArityPhrasePenaltyFF.
-// *
-// * @author Lane Schwartz
-// * @version $LastChangedDate$
-// */
-//public class ArityPhrasePenaltyFFTest {
-//
-// @Test
-// public void alpha() {
-// Assert.assertEquals(ArityPhrasePenaltyFF.ALPHA, - Math.log10(Math.E));
-// }
-//
-// @Test
-// public void estimate() {
-//
-// int featureID = 0;
-// double weight = 0.0;
-// int owner = MonolingualRule.DUMMY_OWNER;
-// int min = 1;
-// int max = 5;
-//
-// ArityPhrasePenaltyFF featureFunction = new ArityPhrasePenaltyFF(featureID, weight, owner, min, max);
-//
-// int lhs = -1;
-// int[] sourceRHS = {24, -1, 42, 738};
-// int[] targetRHS = {-1, 7, 8};
-// float[] featureScores = {-2.35f, -1.78f, -0.52f};
-// int arity = 1;
-//
-// Rule dummyRule = new BilingualRule(lhs, sourceRHS, targetRHS, featureScores, arity);
-//
-// Assert.assertEquals(featureFunction.estimateLogP(dummyRule, -1), ArityPhrasePenaltyFF.ALPHA);
-//
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
index 233a6ed..439e486 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
@@ -26,14 +26,13 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.joshua.corpus.Vocabulary;
-import org.apache.joshua.decoder.ff.lm.berkeley_lm.LMGrammarBerkeley;
import org.apache.joshua.decoder.ff.lm.buildin_lm.TrieLM;
import org.testng.Assert;
import org.testng.annotations.Test;
/**
* Unit tests for testing ARPA language model class.
- *
+ *
* @author Lane Schwartz
*/
public class ArpaFileTest {
@@ -116,7 +115,7 @@ public class ArpaFileTest {
ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
- Map<Integer, Integer> counts = new HashMap<Integer, Integer>();
+ Map<Integer, Integer> counts = new HashMap<>();
boolean iterationOccurred = false;
@@ -125,8 +124,6 @@ public class ArpaFileTest {
iterationOccurred = true;
int order = ngram.order();
- // System.err.println("Order = " + order);
-
int count;
if (counts.containsKey(order)) {
count = counts.get(order) + 1;
@@ -162,65 +159,6 @@ public class ArpaFileTest {
ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
TrieLM lm = new TrieLM(arpaFile);
- // System.err.println(lm.getChildren().size());
Assert.assertNotSame(lm.getChildren().size(), 0);
}
-
- @Test(dependsOnMethods = { "setup", "testIteration", "testChildren" })
- public void testTrie() throws FileNotFoundException {
- ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
-
- TrieLM lm = new TrieLM(arpaFile);
-
- testLm(lm);
-
- }
-
- @Test(dependsOnMethods = { "setup", "testIteration", "testChildren" })
- public void testBerkeley() throws FileNotFoundException {
-
- LMGrammarBerkeley lm = new LMGrammarBerkeley(3, arpaFileName);
-
- testLm(lm);
-
- }
-
- /**
- * @param lm
- */
- private void testLm(NGramLanguageModel lm) {
- // Test unigrams known to be in the language model
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("a")), -1.992672, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("because")), -2.713723, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("boycott")), -4.678545, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of")), -1.609573, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("parliament")), -3.875917, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("potato")), -9.753210, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("resumption")), -4.678545, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("the")), -1.712444, 0.000001f);
-
- // Test unigrams known to NOT be in the language model
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("banana")), -JoshuaConfiguration.lm_ceiling_cost, 0.000001f);
-
- // Test bigrams known to be in the language model
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("because of")), -0.3552987, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of the")), -0.7507797, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("resumption of")), -0.7266324, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("the resumption")), -3.936147, 0.000001f);
-
- // Test trigrams known to be in the language model
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("because of the")), -0.6309999f, 0.000001f);
-
- // Test bigrams know to NOT be in the language model (but the unigrams are)
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("a boycott")), -4.678545f + -0.1195484f, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of parliament")), -3.875917f + -0.1991907f, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("the potato")), -9.753210f + -0.1606644f, 0.000001f);
-// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("potato parliament")), -3.875917f + -0.0f, 0.000001f);
-
- // Test trigrams know to NOT be in the language model (but the bigrams are)
-// int[] words = vocab.getIDs("because of a");
-// double f = lm.ngramLogProbability(words);
-// Assert.assertEquals(f, -1.403534f + -0.03083654f, 0.000001f);
- // //Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of the parliament")), -3.875917f + -0.05237135f, 0.000001f);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
index d541fdc..5e71352 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
@@ -16,20 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.joshua.decoder.ff.lm;
-import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.testng.Assert.assertEquals;
import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.ff.FeatureVector;
import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
public class LanguageModelFFTest {
@@ -37,7 +38,7 @@ public class LanguageModelFFTest {
private LanguageModelFF ff;
- @Before
+ @BeforeMethod
public void setUp() {
Decoder.resetGlobalState();
@@ -49,7 +50,7 @@ public class LanguageModelFFTest {
ff = new LanguageModelFF(weights, args, config);
}
- @After
+ @AfterMethod
public void tearDown() {
Decoder.resetGlobalState();
}
@@ -59,11 +60,11 @@ public class LanguageModelFFTest {
int[] left = {3};
NgramDPState currentState = new NgramDPState(left, new int[left.length]);
- float score = ff.languageModel.sentenceLogProbability(left, 2, 1);
- assertEquals(-99.0f, score, 0.0);
+ float score = ff.getLM().sentenceLogProbability(left, 2, 1);
+ assertEquals(-99.0f, score, 0.0f);
float cost = ff.estimateFutureCost(null, currentState, null);
- assertEquals(score * WEIGHT, cost, 0.0);
+ assertEquals(score * WEIGHT, cost, 0.0f);
}
@Test
@@ -72,11 +73,11 @@ public class LanguageModelFFTest {
int[] left = {startSymbolId};
NgramDPState currentState = new NgramDPState(left, new int[left.length]);
- float score = ff.languageModel.sentenceLogProbability(left, 2, 2);
- assertEquals(0.0f, score, 0.0);
+ float score = ff.getLM().sentenceLogProbability(left, 2, 2);
+ assertEquals(0.0f, score, 0.0f);
float cost = ff.estimateFutureCost(null, currentState, null);
- assertEquals(score * WEIGHT, cost, 0.0);
+ assertEquals(cost, score * WEIGHT, 0.0f);
}
@Test
@@ -86,10 +87,10 @@ public class LanguageModelFFTest {
int[] left = {startSymbolId, 3};
NgramDPState currentState = new NgramDPState(left, new int[left.length]);
- float score = ff.languageModel.sentenceLogProbability(left, 2, 2);
- assertEquals(-100.752754f, score, 0.0f);
+ float score = ff.getLM().sentenceLogProbability(left, 2, 2);
+ assertEquals(score, -100.752754f, 0.0f);
float cost = ff.estimateFutureCost(null, currentState, null);
- assertEquals(score * WEIGHT, cost, 0.0f);
+ assertEquals(cost, score * WEIGHT, 0.0f);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
index 8f6dcbb..b0612d4 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
@@ -18,27 +18,20 @@
*/
package org.apache.joshua.decoder.ff.lm.berkeley_lm;
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.junit.After;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
-
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
/**
* Replacement for test/lm/berkeley/test.sh regression test
*/
-@RunWith(value = Parameterized.class)
+
public class LMGrammarBerkeleyTest {
private static final String INPUT = "the chat-rooms";
@@ -49,24 +42,21 @@ public class LMGrammarBerkeleyTest {
private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
- @Parameters
- public static List<String> lmFiles() {
- return Arrays.asList("resources/berkeley_lm/lm",
- "resources/berkeley_lm/lm.gz",
- "resources/berkeley_lm/lm.berkeleylm",
- "resources/berkeley_lm/lm.berkeleylm.gz");
+ @DataProvider(name = "languageModelFiles")
+ public Object[][] lmFiles() {
+ return new Object[][]{{"resources/berkeley_lm/lm"},
+ {"resources/berkeley_lm/lm.gz"},
+ {"resources/berkeley_lm/lm.berkeleylm"},
+ {"resources/berkeley_lm/lm.berkeleylm.gz"}};
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
decoder.cleanUp();
}
- @Parameter
- public String lmFile;
-
- @Test
- public void verifyLM() {
+ @Test(dataProvider = "languageModelFiles")
+ public void verifyLM(String lmFile) {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.processCommandLineOptions(OPTIONS);
joshuaConfig.features.add("LanguageModel -lm_type berkeleylm -lm_order 2 -lm_file " + lmFile);
@@ -79,7 +69,7 @@ public class LMGrammarBerkeleyTest {
final Sentence sentence = new Sentence(input, 0, joshuaConfig);
return decoder.decode(sentence);
}
-
+
@Test
public void givenLmWithOovFeature_whenDecoder_thenCorrectFeaturesReturned() {
joshuaConfig = new JoshuaConfiguration();
@@ -90,4 +80,6 @@ public class LMGrammarBerkeleyTest {
assertEquals(Decoder.weights.getDenseFeatures().size(), 3);
assertEquals(translation, EXPECTED_OUTPUT_WITH_OOV);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
index f9eb882..5946abd 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
@@ -28,7 +28,7 @@ import org.apache.joshua.decoder.ff.FeatureVector;
import org.apache.joshua.decoder.ff.lm.LanguageModelFF;
import org.apache.joshua.decoder.ff.tm.OwnerMap;
import org.apache.joshua.decoder.ff.tm.Rule;
-import org.testng.SkipException;
+import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -53,12 +53,7 @@ public class ClassBasedLanguageModelTest {
"-class_map", "./src/test/resources/lm/class_lm/class.map" };
JoshuaConfiguration config = new JoshuaConfiguration();
- try {
- ff = new LanguageModelFF(weights, args, config);
- } catch (ExceptionInInitializerError kenLmException) {
- throw new SkipException("Skipping test because KenLM.so/dylib was not found");
- }
-
+ KenLmTestUtil.Guard(() -> ff = new LanguageModelFF(weights, args, config));
}
@AfterMethod
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/io/TranslationRequestStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/io/TranslationRequestStreamTest.java b/src/test/java/org/apache/joshua/decoder/io/TranslationRequestStreamTest.java
new file mode 100644
index 0000000..a09aebb
--- /dev/null
+++ b/src/test/java/org/apache/joshua/decoder/io/TranslationRequestStreamTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.io;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+/**
+ * This class verifies the following behaviors:
+ * <p>
+ * - A blank input, i.e. "", does not cause a translation to be created.
+ * <p>
+ * - A non-blank input that is not followed by a newline, e.g. "1", causes a translation to be
+ * created.
+ * <p>
+ * - An input that contains whitespace or nothing followed by a newline causes a translation to be
+ * created, with "" as the source.
+ */
+
+public class TranslationRequestStreamTest {
+
+ private final JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+
+ @BeforeMethod
+ public void createTranslationRequest() throws Exception {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeMethod
+ protected void setUp() {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterMethod
+ protected void tearDown() throws Exception {
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream#TranslationRequestStream(BufferedReader, JoshuaConfiguration)}.
+ */
+ @Test(enabled = false)
+ public void testTranslationRequest() {
+ fail("Not yet implemented");
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream#size()}.
+ */
+ @Test
+ public void testSize_uponConstruction() {
+ InputStream in = mock(InputStream.class);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(in, Charset.defaultCharset())), joshuaConfiguration);
+ assertEquals(request.size(), 0);
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream#size()}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testSize_1() throws Exception {
+ byte[] data = "1".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ request.next();
+ assertEquals(request.size(), 1);
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream##size()}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testSize_newline() throws Exception {
+ byte[] data = "\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ request.next();
+ assertEquals(request.size(), 1);
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream##size()}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testSize_2newlines() throws Exception {
+ byte[] data = "\n\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ request.next();
+ request.next();
+ assertEquals(request.size(), 2);
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream##next()}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testNext_2Newlines() throws Exception {
+ byte[] data = "\n\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ assertEquals(request.next().source(), "");
+ assertEquals(request.next().source(), "");
+ }
+
+ /**
+ * Test method for {@link org.apache.joshua.decoder.io.TranslationRequestStream##remove()}.
+ */
+ @Test(enabled = false)
+ public void testRemove() {
+ fail("Not yet implemented");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java b/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java
deleted file mode 100644
index 5a1c3ab..0000000
--- a/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder.io;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-
-import org.apache.joshua.decoder.JoshuaConfiguration;
-
-import org.testng.annotations.*;
-import static org.testng.Assert.*;
-import static org.mockito.Mockito.*;
-
-/**
- * This class verifies the following behaviors:
- *
- * - A blank input, i.e. "", does not cause a translation to be created.
- *
- * - A non-blank input that is not followed by a newline, e.g. "1", causes a translation to be
- * created.
- *
- * - An input that contains whitespace or nothing followed by a newline causes a translation to be
- * created, with "" as the source.
- */
-public class TranslationRequestTest {
-
- private final JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
- @BeforeMethod
- public void createTranslationRequest() throws Exception {
- }
-
- /**
- * @throws java.lang.Exception
- */
- @BeforeMethod
- protected void setUp() throws Exception {
- }
-
- /**
- * @throws java.lang.Exception
- */
- @AfterMethod
- protected void tearDown() throws Exception {
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#TranslationRequest(java.io.InputStream)}.
- */
- @Test(enabled = false)
- public void testTranslationRequest() {
- fail("Not yet implemented");
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- */
- @Test(enabled = true)
- public void testSize_uponConstruction() {
- InputStream in = mock(InputStream.class);
- TranslationRequestStream request = new TranslationRequestStream(
- new BufferedReader(new InputStreamReader(in, Charset.defaultCharset())), joshuaConfiguration);
- assertEquals(request.size(), 0);
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- * @throws Exception
- */
- @Test(enabled = true)
- public void testSize_1() throws Exception {
- byte[] data = "1".getBytes();
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequestStream request = new TranslationRequestStream(
- new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
- request.next();
- assertEquals(request.size(), 1);
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- * @throws Exception
- */
- @Test(enabled = true)
- public void testSize_newline() throws Exception {
- byte[] data = "\n".getBytes();
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequestStream request = new TranslationRequestStream(
- new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
- request.next();
- assertEquals(request.size(), 1);
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
- * @throws Exception
- */
- @Test(enabled = true)
- public void testSize_2newlines() throws Exception {
- byte[] data = "\n\n".getBytes();
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequestStream request = new TranslationRequestStream(
- new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
- request.next();
- request.next();
- assertEquals(request.size(), 2);
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#next()}.
- * @throws Exception
- */
- @Test(enabled = true)
- public void testNext_2Newlines() throws Exception {
- byte[] data = "\n\n".getBytes();
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- TranslationRequestStream request = new TranslationRequestStream(
- new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
- assertEquals(request.next().source(), "");
- assertEquals(request.next().source(), "");
- }
-
- /**
- * Test method for {@link joshua.decoder.io.TranslationRequest#remove()}.
- */
- @Test(enabled = false)
- public void testRemove() {
- fail("Not yet implemented");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java b/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
index 44ef35d..5b9db06 100644
--- a/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
+++ b/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
@@ -16,32 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
- package org.apache.joshua.decoder.kbest_extraction;
-
-import static com.google.common.base.Charsets.UTF_8;
-import static java.nio.file.Files.readAllBytes;
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+package org.apache.joshua.decoder.kbest_extraction;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.io.KenLmTestUtil;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
+import static org.testng.Assert.assertEquals;
/**
* Reimplements the kbest extraction regression test
* TODO (fhieber): this test strangely only works with StateMinimizing KenLM.
* This is to be investigated
*/
-@Ignore("re-enable as soon as kenlm native library support will be in place")
+
public class KBestExtractionTest {
private static final String CONFIG = "resources/kbest_extraction/joshua.config";
@@ -51,15 +50,15 @@ public class KBestExtractionTest {
private JoshuaConfiguration joshuaConfig = null;
private Decoder decoder = null;
- @Before
+ @BeforeMethod
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.readConfigFile(CONFIG);
joshuaConfig.outputFormat = "%i ||| %s ||| %c";
- decoder = new Decoder(joshuaConfig, "");
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig, ""));
}
- @After
+ @AfterMethod
public void tearDown() throws Exception {
decoder.cleanUp();
decoder = null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/94e863b8/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java b/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
index 7526b1f..c6512da 100644
--- a/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
+++ b/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
@@ -18,11 +18,13 @@
*/
package org.apache.joshua.decoder.phrase;
-import static org.junit.Assert.*;
+import org.testng.annotations.Test;
import java.util.BitSet;
-import org.junit.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertFalse;
public class CoverageTest {
[3/3] incubator-joshua git commit: Merge branch 'kellen-testng'
Posted by mj...@apache.org.
Merge branch 'kellen-testng'
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/aadb4d68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/aadb4d68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/aadb4d68
Branch: refs/heads/master
Commit: aadb4d687b3de2ebb3a4acce7b7bf9643ebc4e0c
Parents: a169f71 94e863b
Author: Matt Post <po...@cs.jhu.edu>
Authored: Fri Jul 29 13:22:41 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Fri Jul 29 13:22:41 2016 -0400
----------------------------------------------------------------------
pom.xml | 15 +-
resources/kenlm/oilers.kenlm | Bin 0 -> 49011 bytes
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 53 +++---
.../ff/lm/StateMinimizingLanguageModel.java | 6 +-
.../apache/joshua/corpus/CorpusArrayTest.java | 177 ------------------
.../java/org/apache/joshua/corpus/SpanTest.java | 2 -
.../apache/joshua/corpus/VocabularyTest.java | 60 +++---
.../joshua/corpus/vocab/VocabularyTest.java | 180 ------------------
.../ArtificialGrammarAndCorpusCreater.java | 130 -------------
.../joshua/decoder/DecoderThreadTest.java | 172 -----------------
.../joshua/decoder/JoshuaDecoderTest.java | 83 ---------
.../joshua/decoder/TestConfigFileCreater.java | 184 -------------------
.../apache/joshua/decoder/TranslationsTest.java | 87 ---------
.../decoder/ff/ArityPhrasePenaltyFFTest.java | 64 -------
.../joshua/decoder/ff/lm/ArpaFileTest.java | 66 +------
.../decoder/ff/lm/LanguageModelFFTest.java | 33 ++--
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 44 ++---
.../class_lm/ClassBasedLanguageModelTest.java | 9 +-
.../io/TranslationRequestStreamTest.java | 158 ++++++++++++++++
.../decoder/io/TranslationRequestTest.java | 149 ---------------
.../kbest_extraction/KBestExtractionTest.java | 33 ++--
.../joshua/decoder/phrase/CoverageTest.java | 6 +-
.../ConstrainedPhraseDecodingTest.java | 21 +--
.../phrase/decode/PhraseDecodingTest.java | 20 +-
.../java/org/apache/joshua/lattice/ArcTest.java | 2 -
.../apache/joshua/system/AlignmentMapTest.java | 12 +-
.../org/apache/joshua/system/KenLmTest.java | 51 +++--
.../system/MultithreadedTranslationTests.java | 15 +-
.../joshua/system/StructuredOutputTest.java | 13 +-
.../system/StructuredTranslationTest.java | 15 +-
.../org/apache/joshua/util/FormatUtilsTest.java | 8 +-
.../apache/joshua/util/io/KenLmTestUtil.java | 59 ++++++
32 files changed, 420 insertions(+), 1507 deletions(-)
----------------------------------------------------------------------