You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/11/22 18:08:03 UTC
[2/4] incubator-joshua git commit: JOSHUA-317 - SyntaxError: invalid
syntax scripts/training/run_tuner.py AND JOSHUA-318 -
scripts/training/run_tuner.py should enable configurable memory usage when
invioking joshua-decoder
JOSHUA-317 - SyntaxError: invalid syntax scripts/training/run_tuner.py AND JOSHUA-318 - scripts/training/run_tuner.py should enable configurable memory usage when invioking joshua-decoder
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/000298e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/000298e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/000298e5
Branch: refs/heads/master
Commit: 000298e555fbc71315b1d8719f5c3918a2102e5b
Parents: d137f29
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Wed Oct 26 19:11:07 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Wed Oct 26 19:11:07 2016 -0700
----------------------------------------------------------------------
scripts/support/run_bundler.py | 4 ++--
scripts/training/run_tuner.py | 13 +++++++------
2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/000298e5/scripts/support/run_bundler.py
----------------------------------------------------------------------
diff --git a/scripts/support/run_bundler.py b/scripts/support/run_bundler.py
index 0368b1e..43a5db1 100755
--- a/scripts/support/run_bundler.py
+++ b/scripts/support/run_bundler.py
@@ -659,7 +659,7 @@ def collect_operations(opts):
message = (
# Prepend the line number to the error message
'ERROR: Configuration file "{0}" line {1}: {2}'
- .format(opts.config.name, line_num, e.message)
+ .format(opts.config.name, line_num, str(e))
)
e.message = message
raise e
@@ -675,7 +675,7 @@ def collect_operations(opts):
# Prepend the line number to the error message
message = (
'ERROR: Configuration file "{0}" line {1}: {2}'
- .format(opts.config.name, line_num, e.message)
+ .format(opts.config.name, line_num, str(e))
)
e.message = message
raise e
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/000298e5/scripts/training/run_tuner.py
----------------------------------------------------------------------
diff --git a/scripts/training/run_tuner.py b/scripts/training/run_tuner.py
index b080a99..38059fd 100755
--- a/scripts/training/run_tuner.py
+++ b/scripts/training/run_tuner.py
@@ -350,7 +350,8 @@ def get_features(config_file):
output = check_output("%s/bin/joshua-decoder -c %s -show-weights -v 0" % (JOSHUA, config_file), shell=True)
features = []
- for index, item in enumerate(output.split('\n')):
+ for index, item in enumerate(output.split('\n'.encode(encoding='utf_8', errors='strict'))):
+ item = item.decode()
if item != "":
features.append(tuple(item.split()))
return features
@@ -388,7 +389,7 @@ def setup_configs(template, template_dest, target, num_refs, tunedir, command, c
'NUMREFS': num_refs,
'TUNEDIR': tunedir,
'METRIC': metric,
- 'ITERATIONS': `iterations`,
+ 'ITERATIONS': iterations,
'DECODER_COMMAND': command,
'DECODER_CONFIG': config,
'DECODER_OUTPUT': output })
@@ -416,7 +417,7 @@ def run_zmert(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 10)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.zmert.ZMERT -maxMem 4000 %s/mert.config > %s/mert.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.ZMERT.final'),
@@ -430,7 +431,7 @@ def run_pro(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 30)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.pro.PRO %s/pro.config > %s/pro.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.PRO.final'),
@@ -444,7 +445,7 @@ def run_mira(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 5)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.mira.MIRA %s/mira.config > %s/mira.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.MIRA.final'),
@@ -457,7 +458,7 @@ def run_adagrad(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 10)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.adagrad.AdaGrad %s/adagrad.config > %s/adagrad.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.ADAGRAD.final'),