You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/08/29 17:42:37 UTC
[01/10] incubator-joshua git commit: added berkeleyaligner
Repository: incubator-joshua
Updated Branches:
refs/heads/JOSHUA-304 38eebb3b5 -> 2d106df46
refs/heads/master 6d8f68483 -> 2d106df46
added berkeleyaligner
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d3a2291c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d3a2291c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d3a2291c
Branch: refs/heads/master
Commit: d3a2291c004d383c5cb65ab466af582ab6f39d6c
Parents: 25d28fe
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed Aug 24 09:43:01 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Wed Aug 24 09:43:01 2016 -0400
----------------------------------------------------------------------
download-deps.sh | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d3a2291c/download-deps.sh
----------------------------------------------------------------------
diff --git a/download-deps.sh b/download-deps.sh
index 6e4797b..8fbdf69 100755
--- a/download-deps.sh
+++ b/download-deps.sh
@@ -15,3 +15,7 @@ git clone https://github.com/joshua-decoder/giza-pp.git ext/giza-pp
git clone https://github.com/joshua-decoder/symal.git ext/symal
(make -C ext/symal all)
+
+git clone https://github.com/joshua-decoder/berkeleyaligner ext/berkeleyaligner
+(cd ext/berkeleyaligner; ant)
+
[06/10] incubator-joshua git commit: JOSHUA-285 JOSHUA-296 Refactored
threading in order to properly propagate failures and remove custom code
Posted by mj...@apache.org.
JOSHUA-285 JOSHUA-296 Refactored threading in order to properly propagate failures and remove custom code
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d1c9c074
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d1c9c074
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d1c9c074
Branch: refs/heads/JOSHUA-304
Commit: d1c9c074544d72ee5335bdd83fe415b45098ab08
Parents: 762d588
Author: Kellen Sunderland <ke...@amazon.com>
Authored: Mon Aug 29 10:08:10 2016 +0200
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Mon Aug 29 13:52:33 2016 +0200
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 229 ++++---------------
.../apache/joshua/decoder/DecoderThread.java | 10 +-
.../joshua/decoder/JoshuaConfiguration.java | 3 +
.../org/apache/joshua/decoder/Translations.java | 18 ++
.../org/apache/joshua/server/ServerThread.java | 1 -
.../system/MultithreadedTranslationTests.java | 45 +++-
6 files changed, 111 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d1c9c074/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index 2d753c1..bc64fda 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -31,11 +31,13 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
import com.google.common.base.Strings;
-
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.ff.FeatureVector;
import org.apache.joshua.decoder.ff.FeatureFunction;
@@ -83,6 +85,7 @@ import org.slf4j.LoggerFactory;
* @author Zhifei Li, zhifei.work@gmail.com
* @author wren ng thornton wren@users.sourceforge.net
* @author Lane Schwartz dowobeha@users.sourceforge.net
+ * @author Kellen Sunderland kellen.sunderland@gmail.com
*/
public class Decoder {
@@ -109,8 +112,6 @@ public class Decoder {
public static int VERBOSE = 1;
- private BlockingQueue<DecoderThread> threadPool = null;
-
// ===============================================================
// Constructors
// ===============================================================
@@ -147,7 +148,6 @@ public class Decoder {
private Decoder(JoshuaConfiguration joshuaConfiguration) {
this.joshuaConfiguration = joshuaConfiguration;
this.grammars = new ArrayList<>();
- this.threadPool = new ArrayBlockingQueue<>(this.joshuaConfiguration.num_parallel_decoders, true);
this.customPhraseTable = null;
resetGlobalState();
@@ -165,188 +165,70 @@ public class Decoder {
return new Decoder(joshuaConfiguration);
}
- // ===============================================================
- // Public Methods
- // ===============================================================
-
/**
- * This class is responsible for getting sentences from the TranslationRequest and procuring a
- * DecoderThreadRunner to translate it. Each call to decodeAll(TranslationRequest) launches a
- * thread that will read the request's sentences, obtain a DecoderThread to translate them, and
- * then place the Translation in the appropriate place.
- *
- * @author Matt Post <po...@cs.jhu.edu>
+ * This function is the main entry point into the decoder. It translates all the sentences in a
+ * (possibly boundless) set of input sentences. Each request launches its own thread to read the
+ * sentences of the request.
*
+ * @param request the populated {@link TranslationRequestStream}
+ * @throws RuntimeException if any fatal errors occur during translation
+ * @return an iterable, asynchronously-filled list of Translations
*/
- private class RequestParallelizer extends Thread {
- /* Source of sentences to translate. */
- private final TranslationRequestStream request;
+ public Translations decodeAll(TranslationRequestStream request) {
+ Translations results = new Translations(request);
+ CompletableFuture.runAsync(() -> decodeAllAsync(request, results));
+ return results;
+ }
- /* Where to put translated sentences. */
- private final Translations response;
+ private void decodeAllAsync(TranslationRequestStream request,
+ Translations responseStream) {
- RequestParallelizer(TranslationRequestStream request, Translations response) {
- this.request = request;
- this.response = response;
- }
-
- @Override
- public void run() {
- /*
- * Repeatedly get an input sentence, wait for a DecoderThread, and then start a new thread to
- * translate the sentence. We start a new thread (via DecoderRunnerThread) as opposed to
- * blocking, so that the RequestHandler can go on to the next sentence in this request, which
- * allows parallelization across the sentences of the request.
- */
- for (;;) {
+ // Give the threadpool a friendly name to help debuggers
+ final ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("TranslationWorker-%d")
+ .setDaemon(true)
+ .build();
+ ExecutorService executor = Executors.newFixedThreadPool(this.joshuaConfiguration.num_parallel_decoders,
+ threadFactory);
+ try {
+ for (; ; ) {
Sentence sentence = request.next();
if (sentence == null) {
- response.finish();
break;
}
- // This will block until a DecoderThread becomes available.
- DecoderThread thread = Decoder.this.getThread();
- new DecoderThreadRunner(thread, sentence, response).start();
- }
- }
-
- /**
- * Strips the nonterminals from the lefthand side of the rule.
- *
- * @param rule
- * @return
- */
- private String formatRule(Rule rule) {
- String ruleString = "";
- boolean first = true;
- for (int word: rule.getFrench()) {
- if (!first)
- ruleString += " " + Vocabulary.word(word);
- first = false;
- }
-
- ruleString += " |||"; // space will get added with first English word
- first = true;
- for (int word: rule.getEnglish()) {
- if (!first)
- ruleString += " " + Vocabulary.word(word);
- first = false;
- }
-
- // strip of the leading space
- return ruleString.substring(1);
- }
- }
-
- /**
- * Retrieve a thread from the thread pool, blocking until one is available. The blocking occurs in
- * a fair fashion (i.e,. FIFO across requests).
- *
- * @return a thread that can be used for decoding.
- */
- public DecoderThread getThread() {
- try {
- return threadPool.take();
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return null;
- }
-
- /**
- * This class handles running a DecoderThread (which takes care of the actual translation of an
- * input Sentence, returning a Translation object when its done). This is done in a thread so as
- * not to tie up the RequestHandler that launched it, freeing it to go on to the next sentence in
- * the TranslationRequest, in turn permitting parallelization across the sentences of a request.
- *
- * When the decoder thread is finshed, the Translation object is placed in the correct place in
- * the corresponding Translations object that was returned to the caller of
- * Decoder.decodeAll(TranslationRequest).
- *
- * @author Matt Post <po...@cs.jhu.edu>
- */
- private class DecoderThreadRunner extends Thread {
-
- private final DecoderThread decoderThread;
- private final Sentence sentence;
- private final Translations translations;
-
- DecoderThreadRunner(DecoderThread thread, Sentence sentence, Translations translations) {
- this.decoderThread = thread;
- this.sentence = sentence;
- this.translations = translations;
- }
-
- @Override
- public void run() {
- /*
- * Process any found metadata.
- */
-
- /*
- * Use the thread to translate the sentence. Then record the translation with the
- * corresponding Translations object, and return the thread to the pool.
- */
- try {
- Translation translation = decoderThread.translate(this.sentence);
- translations.record(translation);
-
- /*
- * This is crucial! It's what makes the thread available for the next sentence to be
- * translated.
- */
- threadPool.put(decoderThread);
- } catch (Exception e) {
- throw new RuntimeException(String.format(
- "Input %d: FATAL UNCAUGHT EXCEPTION: %s", sentence.id(), e.getMessage()), e);
- // translations.record(new Translation(sentence, null, featureFunctions, joshuaConfiguration));
+ executor.execute(() -> {
+ try {
+ Translation result = decode(sentence);
+ responseStream.record(result);
+ } catch (Throwable ex) {
+ responseStream.propagate(ex);
+ }
+ });
}
+ responseStream.finish();
+ } finally {
+ executor.shutdown();
}
}
- /**
- * This function is the main entry point into the decoder. It translates all the sentences in a
- * (possibly boundless) set of input sentences. Each request launches its own thread to read the
- * sentences of the request.
- *
- * @param request the populated {@link org.apache.joshua.decoder.io.TranslationRequestStream}
- * @throws IOException if there is an error with the input stream or writing the output
- * @return an iterable, asynchronously-filled list of Translations
- */
- public Translations decodeAll(TranslationRequestStream request) throws IOException {
- Translations translations = new Translations(request);
-
- /* Start a thread to handle requests on the input stream */
- new RequestParallelizer(request, translations).start();
-
- return translations;
- }
-
/**
- * We can also just decode a single sentence.
+ * We can also just decode a single sentence in the same thread.
*
* @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @throws RuntimeException if any fatal errors occur during translation
* @return the sentence {@link org.apache.joshua.decoder.Translation}
*/
public Translation decode(Sentence sentence) {
- // Get a thread.
-
try {
- DecoderThread thread = threadPool.take();
- Translation translation = thread.translate(sentence);
- threadPool.put(thread);
-
- return translation;
-
- } catch (InterruptedException e) {
- e.printStackTrace();
+ DecoderThread decoderThread = new DecoderThread(this.grammars, Decoder.weights, this.featureFunctions, joshuaConfiguration);
+ return decoderThread.translate(sentence);
+ } catch (IOException e) {
+ throw new RuntimeException(String.format(
+ "Input %d: FATAL UNCAUGHT EXCEPTION: %s", sentence.id(), e.getMessage()), e);
}
-
- return null;
}
/**
@@ -355,14 +237,6 @@ public class Decoder {
* afterwards gets a fresh start.
*/
public void cleanUp() {
- // shut down DecoderThreads
- for (DecoderThread thread : threadPool) {
- try {
- thread.join();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
resetGlobalState();
}
@@ -393,7 +267,7 @@ public class Decoder {
} else { // models: replace the weight
String[] fds = Regex.spaces.split(line);
- StringBuffer newSent = new StringBuffer();
+ StringBuilder newSent = new StringBuilder();
if (!Regex.floatingNumber.matches(fds[fds.length - 1])) {
throw new IllegalArgumentException("last field is not a number; the field is: "
+ fds[fds.length - 1]);
@@ -543,11 +417,8 @@ public class Decoder {
}
// Create the threads
- for (int i = 0; i < joshuaConfiguration.num_parallel_decoders; i++) {
- this.threadPool.put(new DecoderThread(this.grammars, Decoder.weights,
- this.featureFunctions, joshuaConfiguration));
- }
- } catch (IOException | InterruptedException e) {
+ //TODO: (kellens) see if we need to wait until initialized before decoding
+ } catch (IOException e) {
LOG.warn(e.getMessage(), e);
}
@@ -579,7 +450,7 @@ public class Decoder {
int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
String path = parsedArgs.get("path");
- Grammar grammar = null;
+ Grammar grammar;
if (! type.equals("moses") && ! type.equals("phrase")) {
if (new File(path).isDirectory()) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d1c9c074/src/main/java/org/apache/joshua/decoder/DecoderThread.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/DecoderThread.java b/src/main/java/org/apache/joshua/decoder/DecoderThread.java
index bdbdba0..a6f39b1 100644
--- a/src/main/java/org/apache/joshua/decoder/DecoderThread.java
+++ b/src/main/java/org/apache/joshua/decoder/DecoderThread.java
@@ -49,7 +49,7 @@ import org.slf4j.LoggerFactory;
* @author Zhifei Li, zhifei.work@gmail.com
*/
-public class DecoderThread extends Thread {
+public class DecoderThread {
private static final Logger LOG = LoggerFactory.getLogger(DecoderThread.class);
private final JoshuaConfiguration joshuaConfiguration;
@@ -64,8 +64,9 @@ public class DecoderThread extends Thread {
// ===============================================================
// Constructor
// ===============================================================
+ //TODO: (kellens) why is weights unused?
public DecoderThread(List<Grammar> grammars, FeatureVector weights,
- List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) throws IOException {
+ List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) throws IOException {
this.joshuaConfiguration = joshuaConfiguration;
this.allGrammars = grammars;
@@ -84,11 +85,6 @@ public class DecoderThread extends Thread {
// Methods
// ===============================================================
- @Override
- public void run() {
- // Nothing to do but wait.
- }
-
/**
* Translate a sentence.
*
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d1c9c074/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java b/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
index e6f2955..6fd73ba 100644
--- a/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
+++ b/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
@@ -272,6 +272,9 @@ public class JoshuaConfiguration {
/* Weights overridden from the command line */
public String weight_overwrite = "";
+ /* Timeout in seconds for threads */
+ public long translation_thread_timeout = 30_000;
+
/**
* This method resets the state of JoshuaConfiguration back to the state after initialization.
* This is useful when for example making different calls to the decoder within the same java
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d1c9c074/src/main/java/org/apache/joshua/decoder/Translations.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Translations.java b/src/main/java/org/apache/joshua/decoder/Translations.java
index 1eb859a..b3f3633 100644
--- a/src/main/java/org/apache/joshua/decoder/Translations.java
+++ b/src/main/java/org/apache/joshua/decoder/Translations.java
@@ -20,6 +20,8 @@ package org.apache.joshua.decoder;
import java.util.Iterator;
import java.util.LinkedList;
+
+import com.google.common.base.Throwables;
import org.apache.joshua.decoder.io.TranslationRequestStream;
/**
@@ -50,6 +52,7 @@ public class Translations implements Iterator<Translation>, Iterable<Translation
private boolean spent = false;
private Translation nextTranslation;
+ private Throwable fatalException;
public Translations(TranslationRequestStream request) {
this.request = request;
@@ -144,6 +147,8 @@ public class Translations implements Iterator<Translation>, Iterable<Translation
}
}
+ fatalErrorCheck();
+
/* We now have the sentence and can return it. */
currentID++;
this.nextTranslation = translations.poll();
@@ -155,4 +160,17 @@ public class Translations implements Iterator<Translation>, Iterable<Translation
public Iterator<Translation> iterator() {
return this;
}
+
+ public void propagate(Throwable ex) {
+ synchronized (this) {
+ fatalException = ex;
+ notify();
+ }
+ }
+
+ private void fatalErrorCheck() {
+ if (fatalException != null) {
+ Throwables.propagate(fatalException);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d1c9c074/src/main/java/org/apache/joshua/server/ServerThread.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/server/ServerThread.java b/src/main/java/org/apache/joshua/server/ServerThread.java
index 72caa5f..32c7b91 100644
--- a/src/main/java/org/apache/joshua/server/ServerThread.java
+++ b/src/main/java/org/apache/joshua/server/ServerThread.java
@@ -35,7 +35,6 @@ import java.util.HashMap;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
-import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d1c9c074/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
index 7b1c47f..7a1d9f4 100644
--- a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
+++ b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
@@ -31,27 +31,31 @@ import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.Translations;
import org.apache.joshua.decoder.io.TranslationRequestStream;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.mockito.Mockito;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import static org.mockito.Mockito.doReturn;
import static org.testng.Assert.assertTrue;
/**
* Integration test for multithreaded Joshua decoder tests. Grammar used is a
* toy packed grammar.
*
- * @author kellens
+ * @author Kellen Sunderland kellen.sunderland@gmail.com
*/
public class MultithreadedTranslationTests {
private JoshuaConfiguration joshuaConfig = null;
private Decoder decoder = null;
private static final String INPUT = "A K B1 U Z1 Z2 B2 C";
+ private static final String EXCEPTION_MESSAGE = "This exception should properly propagate";
private int previousLogLevel;
private final static long NANO_SECONDS_PER_SECOND = 1_000_000_000;
- @BeforeMethod
+ @BeforeClass
public void setUp() throws Exception {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.search_algorithm = "cky";
@@ -88,7 +92,7 @@ public class MultithreadedTranslationTests {
Decoder.VERBOSE = 0;
}
- @AfterMethod
+ @AfterClass
public void tearDown() throws Exception {
this.decoder.cleanUp();
this.decoder = null;
@@ -105,7 +109,7 @@ public class MultithreadedTranslationTests {
// should be sufficient to induce concurrent data access for many shared
// data structures.
- @Test
+ @Test()
public void givenPackedGrammar_whenNTranslationsCalledConcurrently_thenReturnNResults() throws IOException {
// GIVEN
@@ -125,7 +129,7 @@ public class MultithreadedTranslationTests {
ByteArrayOutputStream output = new ByteArrayOutputStream();
// WHEN
- // Translate all spans in parallel.
+ // Translate all segments in parallel.
Translations translations = this.decoder.decodeAll(req);
ArrayList<Translation> translationResults = new ArrayList<Translation>();
@@ -146,10 +150,35 @@ public class MultithreadedTranslationTests {
}
final long translationEndTime = System.nanoTime();
- final double pipelineLoadDurationInSeconds = (translationEndTime - translationStartTime) / ((double)NANO_SECONDS_PER_SECOND);
+ final double pipelineLoadDurationInSeconds = (translationEndTime - translationStartTime)
+ / ((double)NANO_SECONDS_PER_SECOND);
System.err.println(String.format("%.2f seconds", pipelineLoadDurationInSeconds));
// THEN
assertTrue(translationResults.size() == inputLines);
}
+
+ @Test(expectedExceptions = RuntimeException.class,
+ expectedExceptionsMessageRegExp = EXCEPTION_MESSAGE)
+ public void givenDecodeAllCalled_whenRuntimeExceptionThrown_thenPropagate() throws IOException {
+ // GIVEN
+ // A spy request stream that will cause an exception to be thrown on a threadpool thread
+ TranslationRequestStream spyReq = Mockito.spy(new TranslationRequestStream(null, joshuaConfig));
+ doReturn(createSentenceSpyWithRuntimeExceptions()).when(spyReq).next();
+
+ // WHEN
+ // Translate all segments in parallel.
+ Translations translations = this.decoder.decodeAll(spyReq);
+
+ ArrayList<Translation> translationResults = new ArrayList<>();
+ for (Translation t: translations)
+ translationResults.add(t);
+ }
+
+ private Sentence createSentenceSpyWithRuntimeExceptions() {
+ Sentence sent = new Sentence(INPUT, 0, joshuaConfig);
+ Sentence spy = Mockito.spy(sent);
+ Mockito.when(spy.target()).thenThrow(new RuntimeException(EXCEPTION_MESSAGE));
+ return spy;
+ }
}
[02/10] incubator-joshua git commit: formatting fixes,
updated path to JAR
Posted by mj...@apache.org.
formatting fixes, updated path to JAR
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/255927db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/255927db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/255927db
Branch: refs/heads/master
Commit: 255927dba585965822c3d35f4d90e35d3199994f
Parents: d3a2291
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed Aug 24 10:07:38 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Wed Aug 24 10:07:38 2016 -0400
----------------------------------------------------------------------
scripts/training/paralign.pl | 2 +-
.../training/templates/alignment/word-align.conf | 18 +++++++++---------
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/255927db/scripts/training/paralign.pl
----------------------------------------------------------------------
diff --git a/scripts/training/paralign.pl b/scripts/training/paralign.pl
index d5159a7..2f04fc1 100755
--- a/scripts/training/paralign.pl
+++ b/scripts/training/paralign.pl
@@ -78,7 +78,7 @@ sub run_berkeley_aligner {
# run the job
$cachepipe->cmd("berkeley-aligner-chunk-$chunkno",
- "java -d64 -Xmx$args{aligner_mem} -jar $JOSHUA/lib/berkeleyaligner.jar ++alignments/$chunkno/word-align.conf",
+ "java -d64 -Xmx$args{aligner_mem} -jar $JOSHUA/ext/berkeleyaligner/distribution/berkeleyaligner.jar ++alignments/$chunkno/word-align.conf",
"alignments/$chunkno/word-align.conf",
"$args{train_dir}/splits/corpus.$args{source}.$chunkno",
"$args{train_dir}/splits/corpus.$args{target}.$chunkno",
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/255927db/scripts/training/templates/alignment/word-align.conf
----------------------------------------------------------------------
diff --git a/scripts/training/templates/alignment/word-align.conf b/scripts/training/templates/alignment/word-align.conf
index 83904d3..5fe3e0c 100644
--- a/scripts/training/templates/alignment/word-align.conf
+++ b/scripts/training/templates/alignment/word-align.conf
@@ -11,8 +11,8 @@
forwardModels MODEL1 HMM
reverseModels MODEL1 HMM
-mode JOINT JOINT
-iters 5 5
+mode JOINT JOINT
+iters 5 5
###############################################
# Execution: Controls output and program flow
@@ -20,9 +20,9 @@ iters 5 5
execDir alignments/<CHUNK>
create
-saveParams false
-numThreads 1
-msPerLine 10000
+saveParams false
+numThreads 1
+msPerLine 10000
alignTraining
#################
@@ -33,10 +33,10 @@ foreignSuffix <SOURCE>
englishSuffix <TARGET>
# Choose the training sources, which can either be directories or files that list files/directories
-trainSources <TRAIN_DIR>/splits/corpus
-sentences MAX
-testSources /dev/null
-overwriteExecDir true
+trainSources <TRAIN_DIR>/splits/corpus
+sentences MAX
+testSources /dev/null
+overwriteExecDir true
#################
# 1-best output
[03/10] incubator-joshua git commit: Merge branch 'master' into
JOSHUA-304
Posted by mj...@apache.org.
Merge branch 'master' into JOSHUA-304
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/fb5d35da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/fb5d35da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/fb5d35da
Branch: refs/heads/master
Commit: fb5d35da0999fe6533b91b6076a4c4032d5d6710
Parents: 255927d 0744ebf
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed Aug 24 15:44:44 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Wed Aug 24 15:44:44 2016 -0400
----------------------------------------------------------------------
CHANGELOG | 7 +
demo/README.md | 2 +-
demo/apache_joshua_logo.png | Bin 0 -> 306617 bytes
demo/apache_joshua_logo_faded.png | Bin 0 -> 309216 bytes
demo/demo.config | 3 +
demo/demo.js | 19 +-
demo/index.html | 37 +-
examples/README.md | 36 +-
pom.xml | 30 +-
resources/berkeley_lm/lm | 16 -
resources/berkeley_lm/lm.berkeleylm | Bin 4294 -> 0 bytes
resources/berkeley_lm/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
resources/berkeley_lm/lm.gz | Bin 162 -> 0 bytes
resources/grammar.glue | 4 -
resources/kbest_extraction/glue-grammar | 3 -
resources/kbest_extraction/grammar | 25 -
resources/kbest_extraction/joshua.config | 27 -
resources/kbest_extraction/lm.gz | Bin 2466496 -> 0 bytes
resources/kbest_extraction/output.gold | 3126 ------------------
resources/kbest_extraction/output.scores.gold | 3126 ------------------
resources/kenlm/oilers.kenlm | Bin 49011 -> 0 bytes
resources/lm_oov/joshua.config | 17 -
resources/phrase_decoder/config | 29 -
resources/phrase_decoder/constrained.config | 28 -
.../phrase_decoder/constrained.output.gold | 5 -
resources/phrase_decoder/lm.1.gz | Bin 2235 -> 0 bytes
resources/phrase_decoder/output.gold | 1 -
resources/phrase_decoder/rules.1.gz | Bin 2998042 -> 0 bytes
resources/wa_grammar | 3 -
resources/wa_grammar.packed/config | 2 -
resources/wa_grammar.packed/encoding | Bin 154 -> 0 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 45 -> 0 bytes
.../wa_grammar.packed/slice_00000.features | Bin 47 -> 0 bytes
resources/wa_grammar.packed/slice_00000.source | Bin 204 -> 0 bytes
resources/wa_grammar.packed/slice_00000.target | Bin 128 -> 0 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
resources/wa_grammar.packed/vocabulary | Bin 238 -> 0 bytes
.../org/apache/joshua/adagrad/AdaGradCore.java | 101 +-
.../org/apache/joshua/adagrad/Optimizer.java | 348 +-
.../org/apache/joshua/corpus/BasicPhrase.java | 2 +-
.../apache/joshua/corpus/ContiguousPhrase.java | 8 +-
.../java/org/apache/joshua/corpus/Phrase.java | 2 +-
.../java/org/apache/joshua/corpus/Span.java | 6 +-
.../org/apache/joshua/corpus/SymbolTable.java | 2 +-
.../org/apache/joshua/corpus/Vocabulary.java | 10 +-
.../joshua/corpus/syntax/ArraySyntaxTree.java | 51 +-
.../apache/joshua/corpus/syntax/SyntaxTree.java | 10 +-
.../org/apache/joshua/decoder/ArgsParser.java | 8 +-
.../java/org/apache/joshua/decoder/BLEU.java | 72 +-
.../java/org/apache/joshua/decoder/Decoder.java | 28 +-
.../apache/joshua/decoder/DecoderThread.java | 2 +-
.../joshua/decoder/JoshuaConfiguration.java | 45 +-
.../joshua/decoder/NbestMinRiskReranker.java | 33 +-
.../joshua/decoder/StructuredTranslation.java | 9 +-
.../decoder/StructuredTranslationFactory.java | 5 +-
.../org/apache/joshua/decoder/Translation.java | 8 +-
.../org/apache/joshua/decoder/Translations.java | 2 +-
.../joshua/decoder/chart_parser/Cell.java | 12 +-
.../joshua/decoder/chart_parser/Chart.java | 51 +-
.../decoder/chart_parser/ComputeNodeResult.java | 8 +-
.../decoder/chart_parser/CubePruneState.java | 20 +-
.../joshua/decoder/chart_parser/DotChart.java | 24 +-
.../joshua/decoder/chart_parser/SourcePath.java | 4 +-
.../decoder/chart_parser/StateConstraint.java | 5 +-
.../joshua/decoder/chart_parser/SuperNode.java | 2 +-
.../joshua/decoder/ff/FeatureFunction.java | 24 +-
.../apache/joshua/decoder/ff/FeatureVector.java | 21 +-
.../joshua/decoder/ff/LabelCombinationFF.java | 2 +-
.../joshua/decoder/ff/LabelSubstitutionFF.java | 8 +-
.../joshua/decoder/ff/LexicalFeatures.java | 2 +-
.../apache/joshua/decoder/ff/OOVPenalty.java | 7 +-
.../apache/joshua/decoder/ff/PhraseModel.java | 2 +-
.../apache/joshua/decoder/ff/PhrasePenalty.java | 4 +-
.../org/apache/joshua/decoder/ff/RuleFF.java | 6 +-
.../decoder/ff/RulePropertiesQuerying.java | 6 +-
.../org/apache/joshua/decoder/ff/RuleShape.java | 2 +-
.../joshua/decoder/ff/SourceDependentFF.java | 4 +-
.../apache/joshua/decoder/ff/SourcePathFF.java | 2 +-
.../apache/joshua/decoder/ff/TargetBigram.java | 13 +-
.../ff/fragmentlm/ConcatenationIterator.java | 10 +-
.../decoder/ff/fragmentlm/FragmentLMFF.java | 59 +-
.../ff/fragmentlm/PennTreebankReader.java | 17 +-
.../joshua/decoder/ff/fragmentlm/Tree.java | 56 +-
.../joshua/decoder/ff/fragmentlm/Trees.java | 8 +-
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 26 +-
.../joshua/decoder/ff/lm/LanguageModelFF.java | 39 +-
.../ff/lm/berkeley_lm/LMGrammarBerkeley.java | 4 +-
.../ff/lm/bloomfilter_lm/BloomFilter.java | 2 +-
.../BloomFilterLanguageModel.java | 18 +-
.../joshua/decoder/ff/lm/buildin_lm/TrieLM.java | 25 +-
.../joshua/decoder/ff/phrase/Distortion.java | 2 +-
.../ff/similarity/EdgePhraseSimilarityFF.java | 17 +-
.../ff/state_maintenance/NgramDPState.java | 6 +-
.../joshua/decoder/ff/tm/AbstractGrammar.java | 12 +-
.../decoder/ff/tm/BasicRuleCollection.java | 2 +-
.../joshua/decoder/ff/tm/CreateGlueGrammar.java | 2 +-
.../joshua/decoder/ff/tm/GrammarReader.java | 2 +-
.../apache/joshua/decoder/ff/tm/OwnerMap.java | 2 +-
.../org/apache/joshua/decoder/ff/tm/Rule.java | 67 +-
.../decoder/ff/tm/SentenceFilteredGrammar.java | 12 +-
.../decoder/ff/tm/format/MosesFormatReader.java | 2 +-
.../ff/tm/hash_based/ExtensionIterator.java | 2 +-
.../tm/hash_based/MemoryBasedBatchGrammar.java | 8 +-
.../decoder/ff/tm/packed/PackedGrammar.java | 87 +-
.../ff/tm/packed/SliceAggregatingTrie.java | 4 +-
.../decoder/hypergraph/AlignedSourceTokens.java | 2 +-
.../decoder/hypergraph/AllSpansWalker.java | 19 +-
.../hypergraph/DefaultInsideOutside.java | 34 +-
.../joshua/decoder/hypergraph/ForestWalker.java | 10 +-
.../GrammarBuilderWalkerFunction.java | 14 +-
.../joshua/decoder/hypergraph/HGNode.java | 54 +-
.../joshua/decoder/hypergraph/HyperEdge.java | 6 +-
.../joshua/decoder/hypergraph/HyperGraph.java | 30 +-
.../decoder/hypergraph/HyperGraphPruning.java | 9 +-
.../decoder/hypergraph/KBestExtractor.java | 51 +-
.../hypergraph/OutputStringExtractor.java | 8 +-
.../hypergraph/StringToTreeConverter.java | 16 +-
.../decoder/hypergraph/ViterbiExtractor.java | 10 +-
.../hypergraph/WordAlignmentExtractor.java | 2 +-
.../decoder/hypergraph/WordAlignmentState.java | 8 +-
.../apache/joshua/decoder/io/JSONMessage.java | 18 +-
.../decoder/io/TranslationRequestStream.java | 6 +-
.../apache/joshua/decoder/phrase/Candidate.java | 38 +-
.../apache/joshua/decoder/phrase/Coverage.java | 2 +-
.../apache/joshua/decoder/phrase/Future.java | 8 +-
.../apache/joshua/decoder/phrase/Header.java | 87 +
.../joshua/decoder/phrase/Hypothesis.java | 5 +-
.../joshua/decoder/phrase/PhraseChart.java | 73 +-
.../joshua/decoder/phrase/PhraseNodes.java | 58 +
.../joshua/decoder/phrase/PhraseTable.java | 4 +-
.../org/apache/joshua/decoder/phrase/Stack.java | 13 +-
.../apache/joshua/decoder/phrase/Stacks.java | 25 +-
.../joshua/decoder/phrase/TargetPhrases.java | 87 -
.../decoder/segment_file/ConstraintRule.java | 4 +-
.../joshua/decoder/segment_file/Sentence.java | 18 +-
.../joshua/decoder/segment_file/Token.java | 9 +-
.../java/org/apache/joshua/pro/PROCore.java | 22 +-
.../org/apache/joshua/server/ServerThread.java | 9 +-
.../LMBerkeleySentenceProbablityTest.java | 4 +-
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 10 +-
.../class_lm/ClassBasedLanguageModelTest.java | 4 +-
.../kbest_extraction/KBestExtractionTest.java | 4 +-
.../phrase/decode/PhraseDecodingTest.java | 14 +-
.../org/apache/joshua/system/KenLmTest.java | 2 +-
.../apache/joshua/system/LmOovFeatureTest.java | 13 +-
.../system/MultithreadedTranslationTests.java | 4 +-
.../joshua/system/StructuredOutputTest.java | 4 +-
.../system/StructuredTranslationTest.java | 4 +-
src/test/resources/berkeley_lm/lm | 16 +
src/test/resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
src/test/resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
src/test/resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
src/test/resources/grammar.glue | 4 +
.../resources/kbest_extraction/glue-grammar | 3 +
src/test/resources/kbest_extraction/grammar | 25 +
.../resources/kbest_extraction/joshua.config | 27 +
src/test/resources/kbest_extraction/lm.gz | Bin 0 -> 2466496 bytes
src/test/resources/kbest_extraction/output.gold | 3126 ++++++++++++++++++
.../kbest_extraction/output.scores.gold | 3126 ++++++++++++++++++
src/test/resources/kenlm/oilers.kenlm | Bin 0 -> 49011 bytes
src/test/resources/lm_oov/joshua.config | 17 +
src/test/resources/phrase_decoder/config | 29 +
.../resources/phrase_decoder/constrained.config | 28 +
.../phrase_decoder/constrained.output.gold | 5 +
src/test/resources/phrase_decoder/lm.1.gz | Bin 0 -> 2235 bytes
src/test/resources/phrase_decoder/output.gold | 1 +
src/test/resources/phrase_decoder/rules.1.gz | Bin 0 -> 2998042 bytes
src/test/resources/wa_grammar | 3 +
src/test/resources/wa_grammar.packed/config | 2 +
src/test/resources/wa_grammar.packed/encoding | Bin 0 -> 154 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 0 -> 45 bytes
.../wa_grammar.packed/slice_00000.features | Bin 0 -> 47 bytes
.../wa_grammar.packed/slice_00000.source | Bin 0 -> 204 bytes
.../wa_grammar.packed/slice_00000.target | Bin 0 -> 128 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
src/test/resources/wa_grammar.packed/vocabulary | Bin 0 -> 238 bytes
176 files changed, 7633 insertions(+), 7619 deletions(-)
----------------------------------------------------------------------
[09/10] incubator-joshua git commit: Merge branch 'master' into
JOSHUA-304
Posted by mj...@apache.org.
Merge branch 'master' into JOSHUA-304
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/2d106df4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/2d106df4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/2d106df4
Branch: refs/heads/JOSHUA-304
Commit: 2d106df46ca6f5c1130adf5c793041e33d8a7f59
Parents: 38eebb3 6d8f684
Author: Matt Post <po...@cs.jhu.edu>
Authored: Mon Aug 29 13:35:56 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Mon Aug 29 13:35:56 2016 -0400
----------------------------------------------------------------------
README.md | 1 +
.../java/org/apache/joshua/decoder/Decoder.java | 235 +++++--------------
.../org/apache/joshua/decoder/DecoderTask.java | 197 ++++++++++++++++
.../apache/joshua/decoder/DecoderThread.java | 201 ----------------
.../joshua/decoder/JoshuaConfiguration.java | 3 +
.../apache/joshua/decoder/JoshuaDecoder.java | 6 +-
.../org/apache/joshua/decoder/Translation.java | 2 +-
.../decoder/TranslationResponseStream.java | 176 ++++++++++++++
.../org/apache/joshua/decoder/Translations.java | 158 -------------
.../joshua/decoder/chart_parser/Chart.java | 2 +-
.../org/apache/joshua/server/ServerThread.java | 11 +-
.../system/MultithreadedTranslationTests.java | 51 +++-
12 files changed, 480 insertions(+), 563 deletions(-)
----------------------------------------------------------------------
[10/10] incubator-joshua git commit: Merge branch 'master' into
JOSHUA-304
Posted by mj...@apache.org.
Merge branch 'master' into JOSHUA-304
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/2d106df4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/2d106df4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/2d106df4
Branch: refs/heads/master
Commit: 2d106df46ca6f5c1130adf5c793041e33d8a7f59
Parents: 38eebb3 6d8f684
Author: Matt Post <po...@cs.jhu.edu>
Authored: Mon Aug 29 13:35:56 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Mon Aug 29 13:35:56 2016 -0400
----------------------------------------------------------------------
README.md | 1 +
.../java/org/apache/joshua/decoder/Decoder.java | 235 +++++--------------
.../org/apache/joshua/decoder/DecoderTask.java | 197 ++++++++++++++++
.../apache/joshua/decoder/DecoderThread.java | 201 ----------------
.../joshua/decoder/JoshuaConfiguration.java | 3 +
.../apache/joshua/decoder/JoshuaDecoder.java | 6 +-
.../org/apache/joshua/decoder/Translation.java | 2 +-
.../decoder/TranslationResponseStream.java | 176 ++++++++++++++
.../org/apache/joshua/decoder/Translations.java | 158 -------------
.../joshua/decoder/chart_parser/Chart.java | 2 +-
.../org/apache/joshua/server/ServerThread.java | 11 +-
.../system/MultithreadedTranslationTests.java | 51 +++-
12 files changed, 480 insertions(+), 563 deletions(-)
----------------------------------------------------------------------
[07/10] incubator-joshua git commit: Renamed DecoderThread to
DecoderTask
Posted by mj...@apache.org.
Renamed DecoderThread to DecoderTask
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/0bb29329
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/0bb29329
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/0bb29329
Branch: refs/heads/JOSHUA-304
Commit: 0bb293295e3670c7449815941566578facd247e9
Parents: d1c9c07
Author: Kellen Sunderland <ke...@amazon.com>
Authored: Mon Aug 29 13:53:53 2016 +0200
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Mon Aug 29 13:53:53 2016 +0200
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 6 +-
.../org/apache/joshua/decoder/DecoderTask.java | 197 +++++++++++++++++++
.../apache/joshua/decoder/DecoderThread.java | 197 -------------------
.../apache/joshua/decoder/JoshuaDecoder.java | 2 +-
.../org/apache/joshua/decoder/Translation.java | 2 +-
.../joshua/decoder/chart_parser/Chart.java | 2 +-
6 files changed, 203 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0bb29329/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index bc64fda..c7b2168 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -76,7 +76,7 @@ import org.slf4j.LoggerFactory;
* but also ensures that round-robin parallelization occurs, since RequestParallelizer uses the
* thread pool before translating each request.
*
- * A decoding thread is handled by DecoderThread and launched from DecoderThreadRunner. The purpose
+ * A decoding thread is handled by DecoderTask and launched from DecoderThreadRunner. The purpose
* of the runner is to record where to place the translated sentence when it is done (i.e., which
* Translations object). Translations itself is an iterator whose next() call blocks until the next
* translation is available.
@@ -223,8 +223,8 @@ public class Decoder {
*/
public Translation decode(Sentence sentence) {
try {
- DecoderThread decoderThread = new DecoderThread(this.grammars, Decoder.weights, this.featureFunctions, joshuaConfiguration);
- return decoderThread.translate(sentence);
+ DecoderTask decoderTask = new DecoderTask(this.grammars, Decoder.weights, this.featureFunctions, joshuaConfiguration);
+ return decoderTask.translate(sentence);
} catch (IOException e) {
throw new RuntimeException(String.format(
"Input %d: FATAL UNCAUGHT EXCEPTION: %s", sentence.id(), e.getMessage()), e);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0bb29329/src/main/java/org/apache/joshua/decoder/DecoderTask.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/DecoderTask.java b/src/main/java/org/apache/joshua/decoder/DecoderTask.java
new file mode 100644
index 0000000..e6ce331
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/DecoderTask.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.chart_parser.Chart;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.SourceDependentFF;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.hypergraph.ForestWalker;
+import org.apache.joshua.decoder.hypergraph.GrammarBuilderWalkerFunction;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.phrase.Stacks;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.corpus.Vocabulary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class handles decoding of individual Sentence objects (which can represent plain sentences
+ * or lattices). A single sentence can be decoded by a call to translate() and, if an InputHandler
+ * is used, many sentences can be decoded in a thread-safe manner via a single call to
+ * translateAll(), which continually queries the InputHandler for sentences until they have all been
+ * consumed and translated.
+ *
+ * The DecoderFactory class is responsible for launching the threads.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+
+public class DecoderTask {
+ private static final Logger LOG = LoggerFactory.getLogger(DecoderTask.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ /*
+ * these variables may be the same across all threads (e.g., just copy from DecoderFactory), or
+ * differ from thread to thread
+ */
+ private final List<Grammar> allGrammars;
+ private final List<FeatureFunction> featureFunctions;
+
+
+ // ===============================================================
+ // Constructor
+ // ===============================================================
+ //TODO: (kellens) why is weights unused?
+ public DecoderTask(List<Grammar> grammars, FeatureVector weights,
+ List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) throws IOException {
+
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.allGrammars = grammars;
+
+ this.featureFunctions = new ArrayList<>();
+ for (FeatureFunction ff : featureFunctions) {
+ if (ff instanceof SourceDependentFF) {
+ this.featureFunctions.add(((SourceDependentFF) ff).clone());
+ } else {
+ this.featureFunctions.add(ff);
+ }
+ }
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ /**
+ * Translate a sentence.
+ *
+ * @param sentence The sentence to be translated.
+ * @return the sentence {@link org.apache.joshua.decoder.Translation}
+ */
+ public Translation translate(Sentence sentence) {
+
+ LOG.info("Input {}: {}", sentence.id(), sentence.fullSource());
+
+ if (sentence.target() != null)
+ LOG.info("Input {}: Constraining to target sentence '{}'",
+ sentence.id(), sentence.target());
+
+ // skip blank sentences
+ if (sentence.isEmpty()) {
+ LOG.info("Translation {}: Translation took 0 seconds", sentence.id());
+ return new Translation(sentence, null, featureFunctions, joshuaConfiguration);
+ }
+
+ long startTime = System.currentTimeMillis();
+
+ int numGrammars = allGrammars.size();
+ Grammar[] grammars = new Grammar[numGrammars];
+
+ for (int i = 0; i < allGrammars.size(); i++)
+ grammars[i] = allGrammars.get(i);
+
+ if (joshuaConfiguration.segment_oovs)
+ sentence.segmentOOVs(grammars);
+
+ /**
+ * Joshua supports (as of September 2014) both phrase-based and hierarchical decoding. Here
+ * we build the appropriate chart. The output of both systems is a hypergraph, which is then
+ * used for further processing (e.g., k-best extraction).
+ */
+ HyperGraph hypergraph = null;
+ try {
+
+ if (joshuaConfiguration.search_algorithm.equals("stack")) {
+ Stacks stacks = new Stacks(sentence, this.featureFunctions, grammars, joshuaConfiguration);
+
+ hypergraph = stacks.search();
+ } else {
+ /* Seeding: the chart only sees the grammars, not the factories */
+ Chart chart = new Chart(sentence, this.featureFunctions, grammars,
+ joshuaConfiguration.goal_symbol, joshuaConfiguration);
+
+ hypergraph = (joshuaConfiguration.use_dot_chart)
+ ? chart.expand()
+ : chart.expandSansDotChart();
+ }
+
+ } catch (java.lang.OutOfMemoryError e) {
+ LOG.error("Input {}: out of memory", sentence.id());
+ hypergraph = null;
+ }
+
+ float seconds = (System.currentTimeMillis() - startTime) / 1000.0f;
+ LOG.info("Input {}: Translation took {} seconds", sentence.id(), seconds);
+ LOG.info("Input {}: Memory used is {} MB", sentence.id(), (Runtime
+ .getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0);
+
+ /* Return the translation unless we're doing synchronous parsing. */
+ if (!joshuaConfiguration.parse || hypergraph == null) {
+ return new Translation(sentence, hypergraph, featureFunctions, joshuaConfiguration);
+ }
+
+ /*****************************************************************************************/
+
+ /*
+ * Synchronous parsing.
+ *
+ * Step 1. Traverse the hypergraph to create a grammar for the second-pass parse.
+ */
+ Grammar newGrammar = getGrammarFromHyperGraph(joshuaConfiguration.goal_symbol, hypergraph);
+ newGrammar.sortGrammar(this.featureFunctions);
+ long sortTime = System.currentTimeMillis();
+ LOG.info("Sentence {}: New grammar has {} rules.", sentence.id(),
+ newGrammar.getNumRules());
+
+ /* Step 2. Create a new chart and parse with the instantiated grammar. */
+ Grammar[] newGrammarArray = new Grammar[] { newGrammar };
+ Sentence targetSentence = new Sentence(sentence.target(), sentence.id(), joshuaConfiguration);
+ Chart chart = new Chart(targetSentence, featureFunctions, newGrammarArray, "GOAL",joshuaConfiguration);
+ int goalSymbol = GrammarBuilderWalkerFunction.goalSymbol(hypergraph);
+ String goalSymbolString = Vocabulary.word(goalSymbol);
+ LOG.info("Sentence {}: goal symbol is {} ({}).", sentence.id(),
+ goalSymbolString, goalSymbol);
+ chart.setGoalSymbolID(goalSymbol);
+
+ /* Parsing */
+ HyperGraph englishParse = chart.expand();
+ long secondParseTime = System.currentTimeMillis();
+ LOG.info("Sentence {}: Finished second chart expansion ({} seconds).",
+ sentence.id(), (secondParseTime - sortTime) / 1000);
+ LOG.info("Sentence {} total time: {} seconds.\n", sentence.id(),
+ (secondParseTime - startTime) / 1000);
+ LOG.info("Memory used after sentence {} is {} MB", sentence.id(), (Runtime
+ .getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0);
+ return new Translation(sentence, englishParse, featureFunctions, joshuaConfiguration); // or do something else
+ }
+
+ private Grammar getGrammarFromHyperGraph(String goal, HyperGraph hg) {
+ GrammarBuilderWalkerFunction f = new GrammarBuilderWalkerFunction(goal,joshuaConfiguration);
+ ForestWalker walker = new ForestWalker();
+ walker.walk(hg.goalNode, f);
+ return f.getGrammar();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0bb29329/src/main/java/org/apache/joshua/decoder/DecoderThread.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/DecoderThread.java b/src/main/java/org/apache/joshua/decoder/DecoderThread.java
deleted file mode 100644
index a6f39b1..0000000
--- a/src/main/java/org/apache/joshua/decoder/DecoderThread.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.joshua.decoder.chart_parser.Chart;
-import org.apache.joshua.decoder.ff.FeatureFunction;
-import org.apache.joshua.decoder.ff.FeatureVector;
-import org.apache.joshua.decoder.ff.SourceDependentFF;
-import org.apache.joshua.decoder.ff.tm.Grammar;
-import org.apache.joshua.decoder.hypergraph.ForestWalker;
-import org.apache.joshua.decoder.hypergraph.GrammarBuilderWalkerFunction;
-import org.apache.joshua.decoder.hypergraph.HyperGraph;
-import org.apache.joshua.decoder.phrase.Stacks;
-import org.apache.joshua.decoder.segment_file.Sentence;
-import org.apache.joshua.corpus.Vocabulary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class handles decoding of individual Sentence objects (which can represent plain sentences
- * or lattices). A single sentence can be decoded by a call to translate() and, if an InputHandler
- * is used, many sentences can be decoded in a thread-safe manner via a single call to
- * translateAll(), which continually queries the InputHandler for sentences until they have all been
- * consumed and translated.
- *
- * The DecoderFactory class is responsible for launching the threads.
- *
- * @author Matt Post post@cs.jhu.edu
- * @author Zhifei Li, zhifei.work@gmail.com
- */
-
-public class DecoderThread {
- private static final Logger LOG = LoggerFactory.getLogger(DecoderThread.class);
-
- private final JoshuaConfiguration joshuaConfiguration;
- /*
- * these variables may be the same across all threads (e.g., just copy from DecoderFactory), or
- * differ from thread to thread
- */
- private final List<Grammar> allGrammars;
- private final List<FeatureFunction> featureFunctions;
-
-
- // ===============================================================
- // Constructor
- // ===============================================================
- //TODO: (kellens) why is weights unused?
- public DecoderThread(List<Grammar> grammars, FeatureVector weights,
- List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) throws IOException {
-
- this.joshuaConfiguration = joshuaConfiguration;
- this.allGrammars = grammars;
-
- this.featureFunctions = new ArrayList<>();
- for (FeatureFunction ff : featureFunctions) {
- if (ff instanceof SourceDependentFF) {
- this.featureFunctions.add(((SourceDependentFF) ff).clone());
- } else {
- this.featureFunctions.add(ff);
- }
- }
- }
-
- // ===============================================================
- // Methods
- // ===============================================================
-
- /**
- * Translate a sentence.
- *
- * @param sentence The sentence to be translated.
- * @return the sentence {@link org.apache.joshua.decoder.Translation}
- */
- public Translation translate(Sentence sentence) {
-
- LOG.info("Input {}: {}", sentence.id(), sentence.fullSource());
-
- if (sentence.target() != null)
- LOG.info("Input {}: Constraining to target sentence '{}'",
- sentence.id(), sentence.target());
-
- // skip blank sentences
- if (sentence.isEmpty()) {
- LOG.info("Translation {}: Translation took 0 seconds", sentence.id());
- return new Translation(sentence, null, featureFunctions, joshuaConfiguration);
- }
-
- long startTime = System.currentTimeMillis();
-
- int numGrammars = allGrammars.size();
- Grammar[] grammars = new Grammar[numGrammars];
-
- for (int i = 0; i < allGrammars.size(); i++)
- grammars[i] = allGrammars.get(i);
-
- if (joshuaConfiguration.segment_oovs)
- sentence.segmentOOVs(grammars);
-
- /**
- * Joshua supports (as of September 2014) both phrase-based and hierarchical decoding. Here
- * we build the appropriate chart. The output of both systems is a hypergraph, which is then
- * used for further processing (e.g., k-best extraction).
- */
- HyperGraph hypergraph = null;
- try {
-
- if (joshuaConfiguration.search_algorithm.equals("stack")) {
- Stacks stacks = new Stacks(sentence, this.featureFunctions, grammars, joshuaConfiguration);
-
- hypergraph = stacks.search();
- } else {
- /* Seeding: the chart only sees the grammars, not the factories */
- Chart chart = new Chart(sentence, this.featureFunctions, grammars,
- joshuaConfiguration.goal_symbol, joshuaConfiguration);
-
- hypergraph = (joshuaConfiguration.use_dot_chart)
- ? chart.expand()
- : chart.expandSansDotChart();
- }
-
- } catch (java.lang.OutOfMemoryError e) {
- LOG.error("Input {}: out of memory", sentence.id());
- hypergraph = null;
- }
-
- float seconds = (System.currentTimeMillis() - startTime) / 1000.0f;
- LOG.info("Input {}: Translation took {} seconds", sentence.id(), seconds);
- LOG.info("Input {}: Memory used is {} MB", sentence.id(), (Runtime
- .getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0);
-
- /* Return the translation unless we're doing synchronous parsing. */
- if (!joshuaConfiguration.parse || hypergraph == null) {
- return new Translation(sentence, hypergraph, featureFunctions, joshuaConfiguration);
- }
-
- /*****************************************************************************************/
-
- /*
- * Synchronous parsing.
- *
- * Step 1. Traverse the hypergraph to create a grammar for the second-pass parse.
- */
- Grammar newGrammar = getGrammarFromHyperGraph(joshuaConfiguration.goal_symbol, hypergraph);
- newGrammar.sortGrammar(this.featureFunctions);
- long sortTime = System.currentTimeMillis();
- LOG.info("Sentence {}: New grammar has {} rules.", sentence.id(),
- newGrammar.getNumRules());
-
- /* Step 2. Create a new chart and parse with the instantiated grammar. */
- Grammar[] newGrammarArray = new Grammar[] { newGrammar };
- Sentence targetSentence = new Sentence(sentence.target(), sentence.id(), joshuaConfiguration);
- Chart chart = new Chart(targetSentence, featureFunctions, newGrammarArray, "GOAL",joshuaConfiguration);
- int goalSymbol = GrammarBuilderWalkerFunction.goalSymbol(hypergraph);
- String goalSymbolString = Vocabulary.word(goalSymbol);
- LOG.info("Sentence {}: goal symbol is {} ({}).", sentence.id(),
- goalSymbolString, goalSymbol);
- chart.setGoalSymbolID(goalSymbol);
-
- /* Parsing */
- HyperGraph englishParse = chart.expand();
- long secondParseTime = System.currentTimeMillis();
- LOG.info("Sentence {}: Finished second chart expansion ({} seconds).",
- sentence.id(), (secondParseTime - sortTime) / 1000);
- LOG.info("Sentence {} total time: {} seconds.\n", sentence.id(),
- (secondParseTime - startTime) / 1000);
- LOG.info("Memory used after sentence {} is {} MB", sentence.id(), (Runtime
- .getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0);
- return new Translation(sentence, englishParse, featureFunctions, joshuaConfiguration); // or do something else
- }
-
- private Grammar getGrammarFromHyperGraph(String goal, HyperGraph hg) {
- GrammarBuilderWalkerFunction f = new GrammarBuilderWalkerFunction(goal,joshuaConfiguration);
- ForestWalker walker = new ForestWalker();
- walker.walk(hg.goalNode, f);
- return f.getGrammar();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0bb29329/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
index 4c31655..b1b9d1e 100644
--- a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
+++ b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
/**
* Implements decoder initialization, including interaction with <code>JoshuaConfiguration</code>
- * and <code>DecoderThread</code>.
+ * and <code>DecoderTask</code>.
*
* @author Zhifei Li, zhifei.work@gmail.com
* @author wren ng thornton wren@users.sourceforge.net
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0bb29329/src/main/java/org/apache/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Translation.java b/src/main/java/org/apache/joshua/decoder/Translation.java
index 1688805..142ff05 100644
--- a/src/main/java/org/apache/joshua/decoder/Translation.java
+++ b/src/main/java/org/apache/joshua/decoder/Translation.java
@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
/**
* This class represents translated input objects (sentences or lattices). It is aware of the source
* sentence and id and contains the decoded hypergraph. Translation objects are returned by
- * DecoderThread instances to the InputHandler, where they are assembled in order for output.
+ * DecoderTask instances to the InputHandler, where they are assembled in order for output.
*
* @author Matt Post post@cs.jhu.edu
* @author Felix Hieber fhieber@amazon.com
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0bb29329/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java b/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
index 5c123f9..bd91a6f 100644
--- a/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
+++ b/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
@@ -108,7 +108,7 @@ public class Chart {
* for the sentence() method, we should just accept a Segment instead of the
* sentence, segmentID, and constraintSpans parameters. We have the symbol
* table already, so we can do the integerization here instead of in
- * DecoderThread. GrammarFactory.getGrammarForSentence will want the
+ * DecoderTask. GrammarFactory.getGrammarForSentence will want the
* integerized sentence as well, but then we'll need to adjust that interface
* to deal with (non-trivial) lattices too. Of course, we get passed the
* grammars too so we could move all of that into here.
[04/10] incubator-joshua git commit: updated format of corpus splits
to get berkeley aligner working again
Posted by mj...@apache.org.
updated format of corpus splits to get berkeley aligner working again
I'm not sure why, but the Berkeley Aligner broke. It seems that the jar
file that used to be included with Joshua was an old version, despite the
fact that the Berkeley Aligner itself hasn't been updated for almost
a decade. This change introduces some minor differences that get it working
again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/38eebb3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/38eebb3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/38eebb3b
Branch: refs/heads/master
Commit: 38eebb3b58375d0da584f470de66df68476ab938
Parents: fb5d35d
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed Aug 24 16:16:29 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Wed Aug 24 16:16:29 2016 -0400
----------------------------------------------------------------------
scripts/training/paralign.pl | 12 ++++++------
scripts/training/pipeline.pl | 17 +++++++----------
.../training/templates/alignment/word-align.conf | 2 +-
3 files changed, 14 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/38eebb3b/scripts/training/paralign.pl
----------------------------------------------------------------------
diff --git a/scripts/training/paralign.pl b/scripts/training/paralign.pl
index 2f04fc1..8b0fd28 100755
--- a/scripts/training/paralign.pl
+++ b/scripts/training/paralign.pl
@@ -54,9 +54,9 @@ sub run_giza {
my ($chunkdir,$chunkno,$do_parallel) = @_;
my $parallel = ($do_parallel == 1) ? "-parallel" : "";
$cachepipe->cmd("giza-$chunkno",
- "rm -f $chunkdir/corpus.0-0.*; $args{giza_trainer} --root-dir $chunkdir -e $args{target}.$chunkno -f $args{source}.$chunkno -corpus $args{train_dir}/splits/corpus -merge $args{giza_merge} $parallel > $chunkdir/giza.log 2>&1",
- "$args{train_dir}/splits/corpus.$args{source}.$chunkno",
- "$args{train_dir}/splits/corpus.$args{target}.$chunkno",
+ "rm -f $chunkdir/corpus.0-0.*; $args{giza_trainer} --root-dir $chunkdir -e $args{target} -f $args{source} -corpus $args{train_dir}/splits/$chunkno/corpus -merge $args{giza_merge} $parallel > $chunkdir/giza.log 2>&1",
+ "$args{train_dir}/splits/$chunkno/corpus.$args{source}",
+ "$args{train_dir}/splits/$chunkno/corpus.$args{target}",
"$chunkdir/model/aligned.$args{giza_merge}");
}
@@ -67,8 +67,8 @@ sub run_berkeley_aligner {
open FROM, $aligner_conf or die "can't read berkeley alignment template";
open TO, ">", "alignments/$chunkno/word-align.conf" or die "can't write to 'alignments/$chunkno/word-align.conf'";
while (<FROM>) {
- s/<SOURCE>/$args{source}.$chunkno/g;
- s/<TARGET>/$args{target}.$chunkno/g;
+ s/<SOURCE>/$args{source}/g;
+ s/<TARGET>/$args{target}/g;
s/<CHUNK>/$chunkno/g;
s/<TRAIN_DIR>/$args{train_dir}/g;
print TO;
@@ -91,5 +91,5 @@ sub run_jacana_aligner {
# run the job
$cachepipe->cmd("jacana-aligner-chunk-$chunkno",
- "java -d64 -Xmx$args{aligner_mem} -DJACANA_HOME=$jacana_home -jar $JOSHUA/lib/jacana-xy.jar -m $jacana_home/resources/model/fr-en.model -src fr -tgt en -a $args{train_dir}/splits/corpus.$args{source}.$chunkno -b $args{train_dir}/splits/corpus.$args{target}.$chunkno -o $chunkdir/training.align");
+ "java -d64 -Xmx$args{aligner_mem} -DJACANA_HOME=$jacana_home -jar $JOSHUA/lib/jacana-xy.jar -m $jacana_home/resources/model/fr-en.model -src fr -tgt en -a $args{train_dir}/splits/$chunkno/corpus.$args{source} -b $args{train_dir}/splits/$chunkno/corpus.$args{target} -o $chunkdir/training.align");
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/38eebb3b/scripts/training/pipeline.pl
----------------------------------------------------------------------
diff --git a/scripts/training/pipeline.pl b/scripts/training/pipeline.pl
index 08933ec..c0e33d3 100755
--- a/scripts/training/pipeline.pl
+++ b/scripts/training/pipeline.pl
@@ -797,8 +797,11 @@ if (! defined $ALIGNMENT) {
if ($chunk != $lastchunk) {
close CHUNK_SOURCE;
close CHUNK_TARGET;
- open CHUNK_SOURCE, ">", "$DATA_DIRS{train}/splits/corpus.$SOURCE.$chunk" or die;
- open CHUNK_TARGET, ">", "$DATA_DIRS{train}/splits/corpus.$TARGET.$chunk" or die;
+
+ mkdir("$DATA_DIRS{train}/splits/$chunk");
+
+ open CHUNK_SOURCE, ">", "$DATA_DIRS{train}/splits/$chunk/corpus.$SOURCE" or die;
+ open CHUNK_TARGET, ">", "$DATA_DIRS{train}/splits/$chunk/corpus.$TARGET" or die;
$lastchunk = $chunk;
}
@@ -817,13 +820,7 @@ if (! defined $ALIGNMENT) {
# $max_aligner_threads /= 2;
# }
- # # With multi-threading, we can use a pool to set up concurrent GIZA jobs on the chunks.
- #
- # TODO: implement this. There appears to be a problem with calling system() in threads.
- #
- # my $pool = new Thread::Pool(Min => 1, Max => $max_aligner_threads);
-
- system("mkdir alignments") unless -d "alignments";
+ mkdir("alignments") unless -d "alignments";
my $aligner_cmd = (
"$SCRIPTDIR/training/paralign.pl "
@@ -875,7 +872,7 @@ if (! defined $ALIGNMENT) {
if ($ALIGNER eq "giza") {
@aligned_files = map { "alignments/$_/model/aligned.$GIZA_MERGE" } (0..$lastchunk);
} elsif ($ALIGNER eq "berkeley") {
- @aligned_files = map { "alignments/$_/training.align" } (0..$lastchunk);
+ @aligned_files = map { "alignments/$_/training.$TARGET-$SOURCE.align" } (0..$lastchunk);
} elsif ($ALIGNER eq "jacana") {
@aligned_files = map { "alignments/$_/training.align" } (0..$lastchunk);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/38eebb3b/scripts/training/templates/alignment/word-align.conf
----------------------------------------------------------------------
diff --git a/scripts/training/templates/alignment/word-align.conf b/scripts/training/templates/alignment/word-align.conf
index 5fe3e0c..1622fb9 100644
--- a/scripts/training/templates/alignment/word-align.conf
+++ b/scripts/training/templates/alignment/word-align.conf
@@ -33,7 +33,7 @@ foreignSuffix <SOURCE>
englishSuffix <TARGET>
# Choose the training sources, which can either be directories or files that list files/directories
-trainSources <TRAIN_DIR>/splits/corpus
+trainSources <TRAIN_DIR>/splits/<CHUNK>
sentences MAX
testSources /dev/null
overwriteExecDir true
[08/10] incubator-joshua git commit: Renamed Translations class to
TranslationResponseStream
Posted by mj...@apache.org.
Renamed Translations class to TranslationResponseStream
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/6d8f6848
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/6d8f6848
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/6d8f6848
Branch: refs/heads/JOSHUA-304
Commit: 6d8f684836ddc25e40bb32f91d24d3f6e5eb745b
Parents: 0bb2932
Author: Kellen Sunderland <ke...@amazon.com>
Authored: Mon Aug 29 13:58:52 2016 +0200
Committer: Kellen Sunderland <ke...@amazon.com>
Committed: Mon Aug 29 13:58:52 2016 +0200
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 12 +-
.../apache/joshua/decoder/JoshuaDecoder.java | 4 +-
.../decoder/TranslationResponseStream.java | 176 +++++++++++++++++++
.../org/apache/joshua/decoder/Translations.java | 176 -------------------
.../org/apache/joshua/server/ServerThread.java | 10 +-
.../system/MultithreadedTranslationTests.java | 10 +-
6 files changed, 194 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6d8f6848/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index c7b2168..682e290 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -66,7 +66,7 @@ import org.slf4j.LoggerFactory;
*
* After initialization, the main entry point to the Decoder object is
* decodeAll(TranslationRequest), which returns a set of Translation objects wrapped in an iterable
- * Translations object. It is important that we support multithreading both (a) across the sentences
+ * TranslationResponseStream object. It is important that we support multithreading both (a) across the sentences
* within a request and (b) across requests, in a round-robin fashion. This is done by maintaining a
* fixed sized concurrent thread pool. When a new request comes in, a RequestParallelizer thread is
* launched. This object iterates over the request's sentences, obtaining a thread from the
@@ -78,7 +78,7 @@ import org.slf4j.LoggerFactory;
*
* A decoding thread is handled by DecoderTask and launched from DecoderThreadRunner. The purpose
* of the runner is to record where to place the translated sentence when it is done (i.e., which
- * Translations object). Translations itself is an iterator whose next() call blocks until the next
+ * TranslationResponseStream object). TranslationResponseStream itself is an iterator whose next() call blocks until the next
* translation is available.
*
* @author Matt Post post@cs.jhu.edu
@@ -172,16 +172,16 @@ public class Decoder {
*
* @param request the populated {@link TranslationRequestStream}
* @throws RuntimeException if any fatal errors occur during translation
- * @return an iterable, asynchronously-filled list of Translations
+ * @return an iterable, asynchronously-filled list of TranslationResponseStream
*/
- public Translations decodeAll(TranslationRequestStream request) {
- Translations results = new Translations(request);
+ public TranslationResponseStream decodeAll(TranslationRequestStream request) {
+ TranslationResponseStream results = new TranslationResponseStream(request);
CompletableFuture.runAsync(() -> decodeAllAsync(request, results));
return results;
}
private void decodeAllAsync(TranslationRequestStream request,
- Translations responseStream) {
+ TranslationResponseStream responseStream) {
// Give the threadpool a friendly name to help debuggers
final ThreadFactory threadFactory = new ThreadFactoryBuilder()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6d8f6848/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
index b1b9d1e..d10de8c 100644
--- a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
+++ b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
@@ -101,14 +101,14 @@ public class JoshuaDecoder {
BufferedReader reader = new BufferedReader(new InputStreamReader(input));
TranslationRequestStream fileRequest = new TranslationRequestStream(reader, joshuaConfiguration);
- Translations translations = decoder.decodeAll(fileRequest);
+ TranslationResponseStream translationResponseStream = decoder.decodeAll(fileRequest);
// Create the n-best output stream
FileWriter nbest_out = null;
if (joshuaConfiguration.n_best_file != null)
nbest_out = new FileWriter(joshuaConfiguration.n_best_file);
- for (Translation translation: translations) {
+ for (Translation translation: translationResponseStream) {
/**
* We need to munge the feature value outputs in order to be compatible with Moses tuners.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6d8f6848/src/main/java/org/apache/joshua/decoder/TranslationResponseStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/TranslationResponseStream.java b/src/main/java/org/apache/joshua/decoder/TranslationResponseStream.java
new file mode 100644
index 0000000..f64df69
--- /dev/null
+++ b/src/main/java/org/apache/joshua/decoder/TranslationResponseStream.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+
+import com.google.common.base.Throwables;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+
+/**
+ * This class represents a streaming sequence of translations. It is returned by the main entry
+ * point to the Decoder object, the call to decodeAll. The translations here are parallel to the
+ * input sentences in the corresponding TranslationRequest object. Because of parallelization, the
+ * translated sentences might be computed out of order. Each Translation is sent to this
+ * TranslationResponseStream object by a DecoderThreadRunner via the record() function, which places the
+ * Translation in the right place. When the next translation in a sequence is available, next() is
+ * notified.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class TranslationResponseStream implements Iterator<Translation>, Iterable<Translation> {
+
+ /* The source sentences to be translated. */
+ private TranslationRequestStream request = null;
+
+ /*
+ * This records the index of the sentence at the head of the underlying list. The iterator's
+ * next() blocks when the value at this position in the translations LinkedList is null.
+ */
+ private int currentID = 0;
+
+ /* The set of translated sentences. */
+ private LinkedList<Translation> translations = null;
+
+ private boolean spent = false;
+
+ private Translation nextTranslation;
+ private Throwable fatalException;
+
+ public TranslationResponseStream(TranslationRequestStream request) {
+ this.request = request;
+ this.translations = new LinkedList<>();
+ }
+
+ /**
+ * This is called when null is received from the TranslationRequest, indicating that there are no
+ * more input sentences to translated. That in turn means that the request size will no longer
+ * grow. We then notify any waiting thread if the last ID we've processed is the last one, period.
+ */
+ public void finish() {
+ synchronized (this) {
+ spent = true;
+ if (currentID == request.size()) {
+ this.notifyAll();
+ }
+ }
+ }
+
+ /**
+ * This is called whenever a translation is completed by one of the decoder threads. There may be
+ * a current output thread waiting for the current translation, which is determined by checking if
+ * the ID of the translation is the same as the one being waited for (currentID). If so, the
+ * thread waiting for it is notified.
+ *
+ * @param translation a translated input object
+ */
+ public void record(Translation translation) {
+ synchronized (this) {
+
+ /* Pad the set of translations with nulls to accommodate the new translation. */
+ int offset = translation.id() - currentID;
+ while (offset >= translations.size())
+ translations.add(null);
+ translations.set(offset, translation);
+
+ /*
+ * If the id of the current translation is at the head of the list (first element), then we
+ * have the next Translation to be return, and we should notify anyone waiting on next(),
+ * which will then remove the item and increment the currentID.
+ */
+ if (translation.id() == currentID) {
+ this.notify();
+ }
+ }
+ }
+
+ /**
+ * Returns the next Translation, blocking if necessary until it's available, since the next
+ * Translation might not have been produced yet.
+ *
+ * @return first element from the list of {@link org.apache.joshua.decoder.Translation}'s
+ */
+ @Override
+ public Translation next() {
+ synchronized(this) {
+ if (this.hasNext()) {
+ Translation t = this.nextTranslation;
+ this.nextTranslation = null;
+ return t;
+ }
+
+ return null;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ synchronized (this) {
+
+ if (nextTranslation != null)
+ return true;
+
+ /*
+ * If there are no more input sentences, and we've already distributed what we then know is
+ * the last one, we're done.
+ */
+ if (spent && currentID == request.size())
+ return false;
+
+ /*
+ * Otherwise, there is another sentence. If it's not available already, we need to wait for
+ * it.
+ */
+ if (translations.size() == 0 || translations.peek() == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ fatalErrorCheck();
+
+ /* We now have the sentence and can return it. */
+ currentID++;
+ this.nextTranslation = translations.poll();
+ return this.nextTranslation != null;
+ }
+ }
+
+ @Override
+ public Iterator<Translation> iterator() {
+ return this;
+ }
+
+ public void propagate(Throwable ex) {
+ synchronized (this) {
+ fatalException = ex;
+ notify();
+ }
+ }
+
+ private void fatalErrorCheck() {
+ if (fatalException != null) {
+ Throwables.propagate(fatalException);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6d8f6848/src/main/java/org/apache/joshua/decoder/Translations.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Translations.java b/src/main/java/org/apache/joshua/decoder/Translations.java
deleted file mode 100644
index b3f3633..0000000
--- a/src/main/java/org/apache/joshua/decoder/Translations.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import com.google.common.base.Throwables;
-import org.apache.joshua.decoder.io.TranslationRequestStream;
-
-/**
- * This class represents a streaming sequence of translations. It is returned by the main entry
- * point to the Decoder object, the call to decodeAll. The translations here are parallel to the
- * input sentences in the corresponding TranslationRequest object. Because of parallelization, the
- * translated sentences might be computed out of order. Each Translation is sent to this
- * Translations object by a DecoderThreadRunner via the record() function, which places the
- * Translation in the right place. When the next translation in a sequence is available, next() is
- * notified.
- *
- * @author Matt Post post@cs.jhu.edu
- */
-public class Translations implements Iterator<Translation>, Iterable<Translation> {
-
- /* The source sentences to be translated. */
- private TranslationRequestStream request = null;
-
- /*
- * This records the index of the sentence at the head of the underlying list. The iterator's
- * next() blocks when the value at this position in the translations LinkedList is null.
- */
- private int currentID = 0;
-
- /* The set of translated sentences. */
- private LinkedList<Translation> translations = null;
-
- private boolean spent = false;
-
- private Translation nextTranslation;
- private Throwable fatalException;
-
- public Translations(TranslationRequestStream request) {
- this.request = request;
- this.translations = new LinkedList<>();
- }
-
- /**
- * This is called when null is received from the TranslationRequest, indicating that there are no
- * more input sentences to translated. That in turn means that the request size will no longer
- * grow. We then notify any waiting thread if the last ID we've processed is the last one, period.
- */
- public void finish() {
- synchronized (this) {
- spent = true;
- if (currentID == request.size()) {
- this.notifyAll();
- }
- }
- }
-
- /**
- * This is called whenever a translation is completed by one of the decoder threads. There may be
- * a current output thread waiting for the current translation, which is determined by checking if
- * the ID of the translation is the same as the one being waited for (currentID). If so, the
- * thread waiting for it is notified.
- *
- * @param translation a translated input object
- */
- public void record(Translation translation) {
- synchronized (this) {
-
- /* Pad the set of translations with nulls to accommodate the new translation. */
- int offset = translation.id() - currentID;
- while (offset >= translations.size())
- translations.add(null);
- translations.set(offset, translation);
-
- /*
- * If the id of the current translation is at the head of the list (first element), then we
- * have the next Translation to be return, and we should notify anyone waiting on next(),
- * which will then remove the item and increment the currentID.
- */
- if (translation.id() == currentID) {
- this.notify();
- }
- }
- }
-
- /**
- * Returns the next Translation, blocking if necessary until it's available, since the next
- * Translation might not have been produced yet.
- *
- * @return first element from the list of {@link org.apache.joshua.decoder.Translation}'s
- */
- @Override
- public Translation next() {
- synchronized(this) {
- if (this.hasNext()) {
- Translation t = this.nextTranslation;
- this.nextTranslation = null;
- return t;
- }
-
- return null;
- }
- }
-
- @Override
- public boolean hasNext() {
- synchronized (this) {
-
- if (nextTranslation != null)
- return true;
-
- /*
- * If there are no more input sentences, and we've already distributed what we then know is
- * the last one, we're done.
- */
- if (spent && currentID == request.size())
- return false;
-
- /*
- * Otherwise, there is another sentence. If it's not available already, we need to wait for
- * it.
- */
- if (translations.size() == 0 || translations.peek() == null) {
- try {
- this.wait();
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- fatalErrorCheck();
-
- /* We now have the sentence and can return it. */
- currentID++;
- this.nextTranslation = translations.poll();
- return this.nextTranslation != null;
- }
- }
-
- @Override
- public Iterator<Translation> iterator() {
- return this;
- }
-
- public void propagate(Throwable ex) {
- synchronized (this) {
- fatalException = ex;
- notify();
- }
- }
-
- private void fatalErrorCheck() {
- if (fatalException != null) {
- Throwables.propagate(fatalException);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6d8f6848/src/main/java/org/apache/joshua/server/ServerThread.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/server/ServerThread.java b/src/main/java/org/apache/joshua/server/ServerThread.java
index 32c7b91..976e543 100644
--- a/src/main/java/org/apache/joshua/server/ServerThread.java
+++ b/src/main/java/org/apache/joshua/server/ServerThread.java
@@ -38,7 +38,7 @@ import com.sun.net.httpserver.HttpHandler;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
-import org.apache.joshua.decoder.Translations;
+import org.apache.joshua.decoder.TranslationResponseStream;
import org.apache.joshua.decoder.ff.tm.Rule;
import org.apache.joshua.decoder.ff.tm.Trie;
import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
@@ -89,11 +89,11 @@ public class ServerThread extends Thread implements HttpHandler {
TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
try {
- Translations translations = decoder.decodeAll(request);
+ TranslationResponseStream translationResponseStream = decoder.decodeAll(request);
OutputStream out = socket.getOutputStream();
- for (Translation translation: translations) {
+ for (Translation translation: translationResponseStream) {
out.write(translation.toString().getBytes());
}
@@ -162,12 +162,12 @@ public class ServerThread extends Thread implements HttpHandler {
BufferedReader reader = new BufferedReader(new StringReader(query));
TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
- Translations translations = decoder.decodeAll(request);
+ TranslationResponseStream translationResponseStream = decoder.decodeAll(request);
JSONMessage message = new JSONMessage();
if (meta != null && ! meta.isEmpty())
handleMetadata(meta, message);
- for (Translation translation: translations) {
+ for (Translation translation: translationResponseStream) {
LOG.info("TRANSLATION: '{}' with {} k-best items", translation, translation.getStructuredTranslations().size());
message.addTranslation(translation);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6d8f6848/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
index 7a1d9f4..8192cb3 100644
--- a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
+++ b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
@@ -29,7 +29,7 @@ import java.util.ArrayList;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
-import org.apache.joshua.decoder.Translations;
+import org.apache.joshua.decoder.TranslationResponseStream;
import org.apache.joshua.decoder.io.TranslationRequestStream;
import org.apache.joshua.decoder.segment_file.Sentence;
import org.mockito.Mockito;
@@ -130,14 +130,14 @@ public class MultithreadedTranslationTests {
// WHEN
// Translate all segments in parallel.
- Translations translations = this.decoder.decodeAll(req);
+ TranslationResponseStream translationResponseStream = this.decoder.decodeAll(req);
ArrayList<Translation> translationResults = new ArrayList<Translation>();
final long translationStartTime = System.nanoTime();
try {
- for (Translation t: translations)
+ for (Translation t: translationResponseStream)
translationResults.add(t);
} finally {
if (output != null) {
@@ -168,10 +168,10 @@ public class MultithreadedTranslationTests {
// WHEN
// Translate all segments in parallel.
- Translations translations = this.decoder.decodeAll(spyReq);
+ TranslationResponseStream translationResponseStream = this.decoder.decodeAll(spyReq);
ArrayList<Translation> translationResults = new ArrayList<>();
- for (Translation t: translations)
+ for (Translation t: translationResponseStream)
translationResults.add(t);
}
[05/10] incubator-joshua git commit: Add Brew install badge to README
Posted by mj...@apache.org.
Add Brew install badge to README
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/762d588e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/762d588e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/762d588e
Branch: refs/heads/JOSHUA-304
Commit: 762d588efc820cc7e6b98fce454b9254d8d15518
Parents: 0744ebf
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Sat Aug 27 18:17:36 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Sat Aug 27 18:17:36 2016 -0700
----------------------------------------------------------------------
README.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/762d588e/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index e0b793d..210b24b 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
[](https://travis-ci.org/apache/incubator-joshua)
+[](http://braumeister.org/formula/joshua)
# Welcome to Apache Joshua (Incubating)
<img src="https://s.apache.org/joshua_logo" align="right" width="300" />