You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/11/22 18:08:02 UTC
[1/4] incubator-joshua git commit: JOSHUA-316 run_bundler.py
returning JOB FAILED (return code 1) TypeError: memoryview: a bytes-like
object is required, not 'str'
Repository: incubator-joshua
Updated Branches:
refs/heads/master bea6161f4 -> 792cab9a2
JOSHUA-316 run_bundler.py returning JOB FAILED (return code 1) TypeError: memoryview: a bytes-like object is required, not 'str'
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d137f297
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d137f297
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d137f297
Branch: refs/heads/master
Commit: d137f2970d1eace1dcec576d287712cd5ffffb88
Parents: 8b530bb
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Tue Oct 25 19:31:10 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Tue Oct 25 19:31:10 2016 -0700
----------------------------------------------------------------------
scripts/language-pack/README.template | 2 +-
scripts/support/run_bundler.py | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d137f297/scripts/language-pack/README.template
----------------------------------------------------------------------
diff --git a/scripts/language-pack/README.template b/scripts/language-pack/README.template
index 03041ca..90a9899 100644
--- a/scripts/language-pack/README.template
+++ b/scripts/language-pack/README.template
@@ -44,7 +44,7 @@ You can then connect directly to the socket using nc or telnet:
cat sentences.txt | prepare.sh | nc localhost 5674 > output.txt
-You can set the RESTful interface by also passing '-server-type http':
+You can set the RESTful interface by also passing `-server-type http`:
joshua -server-port 5674 -server-type http
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d137f297/scripts/support/run_bundler.py
----------------------------------------------------------------------
diff --git a/scripts/support/run_bundler.py b/scripts/support/run_bundler.py
index 734ccb6..0368b1e 100755
--- a/scripts/support/run_bundler.py
+++ b/scripts/support/run_bundler.py
@@ -94,7 +94,7 @@ options that may be useful during decoding include:
complete target hypergraph until it is ready to be processed for output, so
too many simultaneous threads could result in lots of memory usage if a long
sentence results in many sentences being queued up. We have run Joshua with
- as many as 48 threads without any problems of this kind, but it\u2019s useful to
+ as many as 48 threads without any problems of this kind, but it's useful to
keep in the back of your mind.
- `-pop-limit N`
@@ -199,7 +199,7 @@ def filter_through_copy_config_script(config_text, copy_configs):
'Running the copy-config.pl script with the command: ' + cmd
)
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE)
- result, err = p.communicate(config_text)
+ result, err = p.communicate(config_text.encode('utf8'))
if p.returncode != 0:
raise CalledProcessError(
'Encountered an error running the copy-config.pl script.\n'
@@ -207,7 +207,7 @@ def filter_through_copy_config_script(config_text, copy_configs):
' error: %s'
% (cmd, err or '')
)
- return result
+ return result.decode('utf8')
def line_specifies_grammar(line):
@@ -748,7 +748,7 @@ def main(argv):
operations = collect_operations(opts)
execute_operations(operations)
except Exception as e:
- error_quit(e.message)
+ error_quit(str(e))
if __name__ == "__main__":
[3/4] incubator-joshua git commit: Merge branch 'JOSHUA-316' of
github.com:lewismc/incubator-joshua into JOSHUA-316
Posted by mj...@apache.org.
Merge branch 'JOSHUA-316' of github.com:lewismc/incubator-joshua into JOSHUA-316
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/663ea6d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/663ea6d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/663ea6d2
Branch: refs/heads/master
Commit: 663ea6d24c30138d283c0064d261ad0edfa8f689
Parents: bea6161 000298e
Author: Matt Post <po...@cs.jhu.edu>
Authored: Tue Nov 22 13:06:32 2016 -0500
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Tue Nov 22 13:06:32 2016 -0500
----------------------------------------------------------------------
scripts/language-pack/README.template | 2 +-
scripts/support/run_bundler.py | 12 ++++++------
scripts/training/run_tuner.py | 13 +++++++------
3 files changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/663ea6d2/scripts/language-pack/README.template
----------------------------------------------------------------------
diff --cc scripts/language-pack/README.template
index d05b2e1,90a9899..b3bad9b
--- a/scripts/language-pack/README.template
+++ b/scripts/language-pack/README.template
@@@ -48,9 -42,9 +48,9 @@@ optio
You can then connect directly to the socket using nc or telnet:
- cat sentences.txt | prepare.sh | nc localhost 5674 > output.txt
+ cat example.<SRC> | prepare.sh | nc localhost 5674 > output.<TRG>
- You can set the RESTful interface by also passing '-server-type http':
+ You can set the RESTful interface by also passing `-server-type http`:
joshua -server-port 5674 -server-type http
[2/4] incubator-joshua git commit: JOSHUA-317 - SyntaxError: invalid
syntax scripts/training/run_tuner.py AND JOSHUA-318 -
scripts/training/run_tuner.py should enable configurable memory usage when
invioking joshua-decoder
Posted by mj...@apache.org.
JOSHUA-317 - SyntaxError: invalid syntax scripts/training/run_tuner.py AND JOSHUA-318 - scripts/training/run_tuner.py should enable configurable memory usage when invioking joshua-decoder
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/000298e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/000298e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/000298e5
Branch: refs/heads/master
Commit: 000298e555fbc71315b1d8719f5c3918a2102e5b
Parents: d137f29
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Wed Oct 26 19:11:07 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Wed Oct 26 19:11:07 2016 -0700
----------------------------------------------------------------------
scripts/support/run_bundler.py | 4 ++--
scripts/training/run_tuner.py | 13 +++++++------
2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/000298e5/scripts/support/run_bundler.py
----------------------------------------------------------------------
diff --git a/scripts/support/run_bundler.py b/scripts/support/run_bundler.py
index 0368b1e..43a5db1 100755
--- a/scripts/support/run_bundler.py
+++ b/scripts/support/run_bundler.py
@@ -659,7 +659,7 @@ def collect_operations(opts):
message = (
# Prepend the line number to the error message
'ERROR: Configuration file "{0}" line {1}: {2}'
- .format(opts.config.name, line_num, e.message)
+ .format(opts.config.name, line_num, str(e))
)
e.message = message
raise e
@@ -675,7 +675,7 @@ def collect_operations(opts):
# Prepend the line number to the error message
message = (
'ERROR: Configuration file "{0}" line {1}: {2}'
- .format(opts.config.name, line_num, e.message)
+ .format(opts.config.name, line_num, str(e))
)
e.message = message
raise e
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/000298e5/scripts/training/run_tuner.py
----------------------------------------------------------------------
diff --git a/scripts/training/run_tuner.py b/scripts/training/run_tuner.py
index b080a99..38059fd 100755
--- a/scripts/training/run_tuner.py
+++ b/scripts/training/run_tuner.py
@@ -350,7 +350,8 @@ def get_features(config_file):
output = check_output("%s/bin/joshua-decoder -c %s -show-weights -v 0" % (JOSHUA, config_file), shell=True)
features = []
- for index, item in enumerate(output.split('\n')):
+ for index, item in enumerate(output.split('\n'.encode(encoding='utf_8', errors='strict'))):
+ item = item.decode()
if item != "":
features.append(tuple(item.split()))
return features
@@ -388,7 +389,7 @@ def setup_configs(template, template_dest, target, num_refs, tunedir, command, c
'NUMREFS': num_refs,
'TUNEDIR': tunedir,
'METRIC': metric,
- 'ITERATIONS': `iterations`,
+ 'ITERATIONS': iterations,
'DECODER_COMMAND': command,
'DECODER_CONFIG': config,
'DECODER_OUTPUT': output })
@@ -416,7 +417,7 @@ def run_zmert(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 10)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.zmert.ZMERT -maxMem 4000 %s/mert.config > %s/mert.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.ZMERT.final'),
@@ -430,7 +431,7 @@ def run_pro(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 30)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.pro.PRO %s/pro.config > %s/pro.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.PRO.final'),
@@ -444,7 +445,7 @@ def run_mira(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 5)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.mira.MIRA %s/mira.config > %s/mira.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.MIRA.final'),
@@ -457,7 +458,7 @@ def run_adagrad(tunedir, source, target, command, config, output, opts):
target, get_num_refs(target), tunedir, command, config, output,
opts.metric, opts.iterations or 10)
- tuner_mem = '4g'
+ tuner_mem = '10g'
call("java -d64 -Xmx%s -cp %s/target/joshua-*-jar-with-dependencies.jar org.apache.joshua.adagrad.AdaGrad %s/adagrad.config > %s/adagrad.log 2>&1" % (tuner_mem, JOSHUA, tunedir, tunedir), shell=True)
safe_symlink(os.path.join(os.path.dirname(config),'joshua.config.ADAGRAD.final'),
[4/4] incubator-joshua git commit: Merge branch 'JOSHUA-316'
Posted by mj...@apache.org.
Merge branch 'JOSHUA-316'
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/792cab9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/792cab9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/792cab9a
Branch: refs/heads/master
Commit: 792cab9a2f6efc35d370801d241b77c97cce7def
Parents: bea6161 663ea6d
Author: Matt Post <po...@cs.jhu.edu>
Authored: Tue Nov 22 13:07:54 2016 -0500
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Tue Nov 22 13:07:54 2016 -0500
----------------------------------------------------------------------
scripts/language-pack/README.template | 2 +-
scripts/support/run_bundler.py | 12 ++++++------
scripts/training/run_tuner.py | 13 +++++++------
3 files changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------