You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by ma...@apache.org on 2018/03/12 02:05:17 UTC
[2/2] incubator-joshua git commit: Fix: tuner utilize threads when
NUM_THREADS is set
Fix: tuner utilize threads when NUM_THREADS is set
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/97c155f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/97c155f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/97c155f4
Branch: refs/heads/master
Commit: 97c155f4f378504ffd59bfd87766002014ad714f
Parents: cad86aa
Author: Thamme Gowda <tg...@isi.edu>
Authored: Mon Mar 5 17:53:09 2018 -0800
Committer: Thamme Gowda <tg...@isi.edu>
Committed: Mon Mar 5 17:53:09 2018 -0800
----------------------------------------------------------------------
scripts/training/pipeline.pl | 1 +
scripts/training/run_tuner.py | 7 ++++---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/97c155f4/scripts/training/pipeline.pl
----------------------------------------------------------------------
diff --git a/scripts/training/pipeline.pl b/scripts/training/pipeline.pl
index b057dac..c5dc228 100755
--- a/scripts/training/pipeline.pl
+++ b/scripts/training/pipeline.pl
@@ -173,6 +173,7 @@ my $NUM_JOBS = 1;
# The number of threads to use at different pieces in the pipeline
# (giza, decoding)
my $NUM_THREADS = 1;
+$ENV{'NUM_THREADS'} = $NUM_THREADS;
# which LM to use (kenlm or berkeleylm)
my $LM_TYPE = "kenlm";
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/97c155f4/scripts/training/run_tuner.py
----------------------------------------------------------------------
diff --git a/scripts/training/run_tuner.py b/scripts/training/run_tuner.py
index d548aee..6d1f69c 100755
--- a/scripts/training/run_tuner.py
+++ b/scripts/training/run_tuner.py
@@ -347,10 +347,11 @@ def parse_tm_line(line):
def get_features(config_file):
"""Queries the decoder for all dense features that will be fired by the feature
functions activated in the config file"""
-
- mem_size = os.environ.get('JOSHUA_MEM', None)
- mem_arg = '-m %s' % mem_size if mem_size else ''
+
+ mem_arg = '-m %s' % os.environ['JOSHUA_MEM'] if 'JOSHUA_MEM' in os.environ else ''
decode_cmd = "%s/bin/joshua-decoder %s -c %s -show-weights -v 0" % (JOSHUA, mem_arg, config_file)
+ if 'NUM_THREADS' in os.environ:
+ decode_cmd += ' -threads %s' % os.environ['NUM_THREADS']
output = check_output(decode_cmd, shell=True)
features = []
for index, item in enumerate(output.split('\n'.encode(encoding='utf_8', errors='strict'))):