You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by sa...@apache.org on 2017/05/18 21:52:20 UTC
aurora git commit: Added 'aurora task scp' command for
copying/retrieving files to the sandbox of a task instance.
Repository: aurora
Updated Branches:
refs/heads/master b00a15e48 -> 4c0974bc4
Added 'aurora task scp' command for copying/retrieving files to the sandbox of a task instance.
This command essentially mimics scp but expands task instances into their respective user@host:path
For 'aurora task scp' the sandbox is the relative root. However, you can still use absolute paths
(ex. /tmp or /var/log). Tilde expansion is not supported (ex. paths like ~/some/dir will not work)
as they will try to access home directories: in this case, the command will return an error.
Example usage:
>From host to task sandbox folder: `aurora task scp ~/test.txt cluster/role/env/job/instance:`
>From task sandbox folder to host: `aurora task scp cluster/role/env/job/instance:test.txt .`
>From task tmp folder to host: `aurora task scp cluster/role/env/job/instance:/tmp/test.txt .`
>From one task to another task: `aurora task scp cluster/role/env/job/instance:test.txt cluster/role/env/job/instance:some/dir/`
Testing Done:
`./pants test src/test/python/apache/aurora/client/cli:cli`
```
23:07:10 00:03 [run]
============== test session starts ===============
platform darwin -- Python 2.7.10 -- py-1.4.33 -- pytest-2.6.4
plugins: cov, timeout
collected 179 items
src/test/python/apache/aurora/client/cli/test_config_noun.py ...
src/test/python/apache/aurora/client/cli/test_context.py ........
src/test/python/apache/aurora/client/cli/test_version.py .
src/test/python/apache/aurora/client/cli/test_quota.py .....
src/test/python/apache/aurora/client/cli/test_plugins.py .
src/test/python/apache/aurora/client/cli/test_client.py ..
src/test/python/apache/aurora/client/cli/test_sla.py .....
src/test/python/apache/aurora/client/cli/test_open.py .....
src/test/python/apache/aurora/client/cli/test_supdate.py .......................................
src/test/python/apache/aurora/client/cli/test_restart.py ..........
src/test/python/apache/aurora/client/cli/test_status.py .............
src/test/python/apache/aurora/client/cli/test_add.py ....
src/test/python/apache/aurora/client/cli/test_diff.py ..
src/test/python/apache/aurora/client/cli/test_cron.py ..........
src/test/python/apache/aurora/client/cli/test_command_hooks.py ..
src/test/python/apache/aurora/client/cli/test_options.py ......
src/test/python/apache/aurora/client/cli/test_task.py ...............
src/test/python/apache/aurora/client/cli/test_create.py ..............
src/test/python/apache/aurora/client/cli/test_kill.py ......................
src/test/python/apache/aurora/client/cli/test_inspect.py ....
src/test/python/apache/aurora/client/cli/test_api_from_cli.py ..
src/test/python/apache/aurora/client/cli/test_diff_formatter.py ......
========== 179 passed in 24.88 seconds ===========
23:07:37 00:30 [complete]
SUCCESS
```
I've also compiled it within the local cluster with Vagrant and used the command to transfer a text file between the scheduler machine and job I created.
Bugs closed: AURORA-1925
Reviewed at https://reviews.apache.org/r/59163/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/4c0974bc
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/4c0974bc
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/4c0974bc
Branch: refs/heads/master
Commit: 4c0974bc4f1b7b6857f8a4315aac184fd9e1a157
Parents: b00a15e
Author: Jordan Ly <jo...@gmail.com>
Authored: Thu May 18 14:49:23 2017 -0700
Committer: Santhosh Kumar <ss...@twitter.com>
Committed: Thu May 18 14:49:23 2017 -0700
----------------------------------------------------------------------
RELEASE-NOTES.md | 4 +
docs/reference/client-commands.md | 13 ++
.../python/apache/aurora/client/cli/options.py | 11 +-
.../python/apache/aurora/client/cli/task.py | 97 ++++++++++-
.../apache/aurora/client/cli/test_task.py | 163 ++++++++++++++++++-
.../sh/org/apache/aurora/e2e/test_end_to_end.sh | 56 +++++++
6 files changed, 340 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/4c0974bc/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 4e930fb..77376e4 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -21,6 +21,10 @@
in clusters with high contention for resources. Disabled by default, but can be enabled with
enable_update_affinity option, and the reservation timeout can be controlled via
update_affinity_reservation_hold_time.
+- Add `task scp` command to the CLI client for easy transferring of files to/from/between task
+ instances. See [here](docs/reference/client-commands.md#scping-with-task-machines) for details.
+ Currently only fully supported for Mesos containers (you can copy files from the Docker container
+ sandbox but you cannot send files to it).
0.17.0
======
http://git-wip-us.apache.org/repos/asf/aurora/blob/4c0974bc/docs/reference/client-commands.md
----------------------------------------------------------------------
diff --git a/docs/reference/client-commands.md b/docs/reference/client-commands.md
index 582c96a..7a88ccb 100644
--- a/docs/reference/client-commands.md
+++ b/docs/reference/client-commands.md
@@ -25,6 +25,7 @@ Aurora Client Commands
- [Getting Job Status](#getting-job-status)
- [Opening the Web UI](#opening-the-web-ui)
- [SSHing to a Specific Task Machine](#sshing-to-a-specific-task-machine)
+ - [SCPing with Specific Task Machines](#scping-with-specific-task-machines)
- [Templating Command Arguments](#templating-command-arguments)
Introduction
@@ -299,6 +300,18 @@ assigned a particular Job/shard number. This may be useful for quickly
diagnosing issues such as performance issues or abnormal behavior on a
particular machine.
+### SCPing with Specific Task Machines
+
+ aurora task scp [<cluster>/<role>/<env>/<job_name>/<instance_id>]:source [<cluster>/<role>/<env>/<job_name>/<instance_id>]:dest
+
+You can have the Aurora client copy file(s)/folder(s) to, from, and between
+individual tasks. The sandbox folder serves as the relative root and is the
+same folder you see when you browse `chroot` from the Scheduler task UI. You
+can also use absolute paths (like for `/tmp`), but tilde expansion is not
+supported. Currently, this command is only fully supported for Mesos
+containers. Users may use this to copy files from Docker containers but they
+cannot copy files to them.
+
### Templating Command Arguments
aurora task run [-e] [-t THREADS] <job_key> -- <<command-line>>
http://git-wip-us.apache.org/repos/asf/aurora/blob/4c0974bc/src/main/python/apache/aurora/client/cli/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/options.py b/src/main/python/apache/aurora/client/cli/options.py
index 1168703..9fcbf4f 100644
--- a/src/main/python/apache/aurora/client/cli/options.py
+++ b/src/main/python/apache/aurora/client/cli/options.py
@@ -260,7 +260,7 @@ MAX_TOTAL_FAILURES_OPTION = CommandOption('--max-total-failures', type=int, defa
NO_BATCHING_OPTION = CommandOption('--no-batching', default=False, action='store_true',
- help='Run the command on all instances at once, instead of running in batches')
+ help='Run the command on all instances at once, instead of running in batches')
ROLE_ARGUMENT = CommandOption('role', type=parse_qualified_role, metavar='CLUSTER/NAME',
@@ -269,6 +269,15 @@ ROLE_ARGUMENT = CommandOption('role', type=parse_qualified_role, metavar='CLUSTE
ROLE_OPTION = CommandOption('--role', metavar='ROLENAME', default=None,
help='Name of the user/role')
+SCP_OPTIONS = CommandOption('--scp-options', type=parse_options, dest='scp_options',
+ default=None, metavar='scp_options', help='A string of space separated system scp options.')
+
+SCP_SOURCE_ARGUMENT = CommandOption('source', nargs='+', metavar='source',
+ help='Source file(s)/folder(s) to copy, in `[CLUSTER/ROLE/ENV/NAME/INSTANCE:]source` format.')
+
+SCP_DEST_ARGUMENT = CommandOption('dest', metavar='dest',
+ help='Destination path to copy into, in `[CLUSTER/ROLE/ENV/NAME/INSTANCE:]dest` format.')
+
SSH_INSTANCE_ARGUMENT = create_instance_argument(
help_text=('Fully specified job instance key, in CLUSTER/ROLE/ENV/NAME[/INSTANCES] format. '
'If INSTANCES is omitted, a random instance will picked up for the SSH session'))
http://git-wip-us.apache.org/repos/asf/aurora/blob/4c0974bc/src/main/python/apache/aurora/client/cli/task.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/task.py b/src/main/python/apache/aurora/client/cli/task.py
index 370dd7f..652a545 100644
--- a/src/main/python/apache/aurora/client/cli/task.py
+++ b/src/main/python/apache/aurora/client/cli/task.py
@@ -17,7 +17,11 @@
from __future__ import print_function
+import os
import subprocess
+from argparse import ArgumentTypeError
+
+from pystachio import Environment, String
from apache.aurora.client.api.command_runner import (
DistributedCommandRunner,
@@ -30,12 +34,17 @@ from apache.aurora.client.cli.options import (
ALL_INSTANCES,
EXECUTOR_SANDBOX_OPTION,
INSTANCES_SPEC_ARGUMENT,
+ SCP_DEST_ARGUMENT,
+ SCP_OPTIONS,
+ SCP_SOURCE_ARGUMENT,
SSH_INSTANCE_ARGUMENT,
SSH_OPTIONS,
SSH_USER_OPTION,
- CommandOption
+ CommandOption,
+ parse_task_instance_key
)
from apache.aurora.common.clusters import CLUSTERS
+from apache.thermos.config.schema import ThermosContext
class RunCommand(Verb):
@@ -148,6 +157,91 @@ class SshCommand(Verb):
return subprocess.call(ssh_command)
+class ScpCommand(Verb):
+
+ JOB_NOT_FOUND_ERROR_MSG = 'Job or instance %s/%s not found'
+ TILDE_USAGE_ERROR_MSG = 'Command does not support tilde expansion for path: %s'
+
+ @property
+ def name(self):
+ return 'scp'
+
+ @property
+ def help(self):
+ return """executes an scp to/from/between task instance(s). The task sandbox acts as
+ the relative root.
+ """
+
+ @staticmethod
+ def _extract_task_instance_and_path(context, file_path):
+ key = file_path.split(':', 1)
+ try:
+ if (len(key) == 1):
+ return (None, key[0]) # No jobkey specified
+ return (parse_task_instance_key(key[0]), key[1])
+ except ArgumentTypeError as e:
+ raise context.CommandError(EXIT_INVALID_PARAMETER, str(e))
+
+ @staticmethod
+ def _build_path(context, target):
+ (task_instance, path) = ScpCommand._extract_task_instance_and_path(context, target)
+
+ # No jobkey is specified therefore we are using a local path.
+ if (task_instance is None):
+ return path
+
+ # Jobkey specified, we want to convert to the user@host:file scp format
+ (cluster, role, env, name) = task_instance.jobkey
+ instance = set([task_instance.instance])
+ api = context.get_api(cluster)
+ resp = api.query(api.build_query(role, name, env=env, instances=instance))
+ context.log_response_and_raise(resp,
+ err_msg=('Unable to get information about instance: %s' % combine_messages(resp)))
+ if (resp.result.scheduleStatusResult.tasks is None or
+ len(resp.result.scheduleStatusResult.tasks) == 0):
+ raise context.CommandError(EXIT_INVALID_PARAMETER,
+ ScpCommand.JOB_NOT_FOUND_ERROR_MSG % (task_instance.jobkey, task_instance.instance))
+ first_task = resp.result.scheduleStatusResult.tasks[0]
+ assigned = first_task.assignedTask
+ role = assigned.task.job.role
+ slave_host = assigned.slaveHost
+
+ # If path is absolute, use that. Else if it is a tilde expansion, throw an error.
+ # Otherwise, use sandbox as relative root.
+ normalized_input_path = os.path.normpath(path)
+ if (os.path.isabs(normalized_input_path)):
+ final_path = normalized_input_path
+ elif (normalized_input_path.startswith('~/') or normalized_input_path == '~'):
+ raise context.CommandError(EXIT_INVALID_PARAMETER, ScpCommand.TILDE_USAGE_ERROR_MSG % path)
+ else:
+ sandbox_path_pre_format = DistributedCommandRunner.thermos_sandbox(
+ api.cluster,
+ executor_sandbox=context.options.executor_sandbox)
+ thermos_namespace = ThermosContext(
+ task_id=assigned.taskId,
+ ports=assigned.assignedPorts)
+ sandbox_path = String(sandbox_path_pre_format) % Environment(thermos=thermos_namespace)
+ # Join the individual folders to the sandbox path to build safely
+ final_path = os.path.join(str(sandbox_path), *normalized_input_path.split(os.sep))
+
+ return '%s@%s:%s' % (role, slave_host, final_path)
+
+ def get_options(self):
+ return [
+ SCP_SOURCE_ARGUMENT,
+ SCP_DEST_ARGUMENT,
+ SCP_OPTIONS,
+ EXECUTOR_SANDBOX_OPTION
+ ]
+
+ def execute(self, context):
+ scp_command = ['scp']
+ scp_command += context.options.scp_options if context.options.scp_options else []
+ scp_command += [ScpCommand._build_path(context, p) for p in context.options.source]
+ scp_command += [ScpCommand._build_path(context, context.options.dest)]
+ return subprocess.call(scp_command)
+
+
class Task(Noun):
@property
def name(self):
@@ -165,3 +259,4 @@ class Task(Noun):
super(Task, self).__init__()
self.register_verb(RunCommand())
self.register_verb(SshCommand())
+ self.register_verb(ScpCommand())
http://git-wip-us.apache.org/repos/asf/aurora/blob/4c0974bc/src/test/python/apache/aurora/client/cli/test_task.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_task.py b/src/test/python/apache/aurora/client/cli/test_task.py
index 390993f..186cb27 100644
--- a/src/test/python/apache/aurora/client/cli/test_task.py
+++ b/src/test/python/apache/aurora/client/cli/test_task.py
@@ -14,12 +14,14 @@
import contextlib
+import pytest
from mock import Mock, patch
-from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, EXIT_OK
+from apache.aurora.client.cli import EXIT_INVALID_PARAMETER, EXIT_OK, Context
from apache.aurora.client.cli.client import AuroraCommandLine
+from apache.aurora.client.cli.task import ScpCommand
-from .util import AuroraClientCommandTest
+from .util import AuroraClientCommandTest, FakeAuroraCommandContext, mock_verb_options
from gen.apache.aurora.api.ttypes import (
JobKey,
@@ -209,3 +211,160 @@ class TestSshCommand(AuroraClientCommandTest):
result = cmd.execute(['task', 'ssh', 'west/bozo/test/hello', '--command=ls'])
assert result == EXIT_INVALID_PARAMETER
assert mock_subprocess.call_count == 0
+
+
+class TestScpCommand(AuroraClientCommandTest):
+
+ @classmethod
+ def create_status_response(cls):
+ resp = cls.create_simple_success_response()
+ resp.result = Result(
+ scheduleStatusResult=ScheduleStatusResult(tasks=cls.create_scheduled_tasks()))
+ return resp
+
+ @classmethod
+ def create_nojob_status_response(cls):
+ resp = cls.create_simple_success_response()
+ resp.result = Result(scheduleStatusResult=ScheduleStatusResult(tasks=[]))
+ return resp
+
+ def setUp(self):
+ self._command = ScpCommand()
+ self._mock_options = mock_verb_options(self._command)
+ self._fake_context = FakeAuroraCommandContext()
+ self._fake_context.set_options(self._mock_options)
+ self._mock_api = self._fake_context.get_api('UNUSED')
+ self._sandbox_args = {'slave_root': '/slaveroot', 'slave_run_directory': 'slaverun'}
+
+ def test_successful_scp_simple(self):
+ """Test the scp command."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.scp_options = ['-v', '-t']
+ self._mock_options.source = ['test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:./test/dir'
+
+ with contextlib.nested(
+ patch('apache.aurora.client.api.command_runner.DistributedCommandRunner.sandbox_args',
+ return_value=self._sandbox_args),
+ patch('subprocess.call', return_value=EXIT_OK)) as [
+ mock_runner_args_patch,
+ mock_subprocess]:
+ assert self._command.execute(self._fake_context) == EXIT_OK
+ mock_subprocess.assert_called_with(['scp', '-v', '-t', 'test.txt',
+ 'bozo@slavehost:'
+ '/slaveroot/slaves/*/frameworks/*/executors/thermos-1287391823/runs/slaverun/sandbox/'
+ 'test/dir'])
+
+ def test_successful_scp_absolute_path(self):
+ """Test the scp command uses absolute paths correctly."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.scp_options = ['-v']
+ self._mock_options.source = ['test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:/tmp'
+
+ with contextlib.nested(
+ patch('apache.aurora.client.api.command_runner.DistributedCommandRunner.sandbox_args',
+ return_value=self._sandbox_args),
+ patch('subprocess.call', return_value=EXIT_OK)) as [
+ mock_runner_args_patch,
+ mock_subprocess]:
+ assert self._command.execute(self._fake_context) == EXIT_OK
+ mock_subprocess.assert_called_with(['scp', '-v', 'test.txt', 'bozo@slavehost:/tmp'])
+
+ def test_successful_scp_two_instances(self):
+ """Test that the scp command correctly evaluates commands with two jobkeys."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.scp_options = ['-v']
+ self._mock_options.source = ['west/bozo/test/hello/1:test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:/tmp'
+
+ with contextlib.nested(
+ patch('apache.aurora.client.api.command_runner.DistributedCommandRunner.sandbox_args',
+ return_value=self._sandbox_args),
+ patch('subprocess.call', return_value=EXIT_OK)) as [
+ mock_runner_args_patch,
+ mock_subprocess]:
+ assert self._command.execute(self._fake_context) == EXIT_OK
+ mock_subprocess.assert_called_with(['scp', '-v',
+ 'bozo@slavehost:'
+ '/slaveroot/slaves/*/frameworks/*/executors/thermos-1287391823/runs/slaverun/sandbox/'
+ 'test.txt',
+ 'bozo@slavehost:/tmp'])
+
+ def test_successful_scp_multiple_files(self):
+ """Test that the scp command correctly evaluates commands with multiple files."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.source = ['test.txt', 'another.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:test/dir'
+
+ with contextlib.nested(
+ patch('apache.aurora.client.api.command_runner.DistributedCommandRunner.sandbox_args',
+ return_value=self._sandbox_args),
+ patch('subprocess.call', return_value=EXIT_OK)) as [
+ mock_runner_args_patch,
+ mock_subprocess]:
+ assert self._command.execute(self._fake_context) == EXIT_OK
+ mock_subprocess.assert_called_with(['scp', 'test.txt', 'another.txt',
+ 'bozo@slavehost:'
+ '/slaveroot/slaves/*/frameworks/*/executors/thermos-1287391823/runs/slaverun/sandbox/'
+ 'test/dir'])
+
+ def test_scp_invalid_tilde_expansion(self):
+ """Test the scp command fails when using tilde expansion."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.scp_options = ['-v']
+ self._mock_options.source = ['test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:~/test.txt'
+
+ with contextlib.nested(patch('subprocess.call', return_value=EXIT_OK)) as [mock_subprocess]:
+ with pytest.raises(Context.CommandError) as exc:
+ assert self._command.execute(self._fake_context) == EXIT_INVALID_PARAMETER
+ assert(ScpCommand.TILDE_USAGE_ERROR_MSG % '~/test.txt' in exc.value.message)
+ assert mock_subprocess.call_count == 0
+
+ # Test another tilde expansion form
+ self._mock_options.source = ['test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:~'
+
+ with contextlib.nested(patch('subprocess.call', return_value=EXIT_OK)) as [mock_subprocess]:
+ with pytest.raises(Context.CommandError) as exc:
+ assert self._command.execute(self._fake_context) == EXIT_INVALID_PARAMETER
+ assert(ScpCommand.TILDE_USAGE_ERROR_MSG % '~' in exc.value.message)
+ assert mock_subprocess.call_count == 0
+
+ def test_scp_bad_jobkey_no_instance(self):
+ """Test the scp command fails when instance id is not specified."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.source = ['test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello:test/dir'
+
+ with contextlib.nested(patch('subprocess.call', return_value=EXIT_OK)) as [mock_subprocess]:
+ with pytest.raises(Context.CommandError) as exc:
+ assert self._command.execute(self._fake_context) == EXIT_INVALID_PARAMETER
+ assert('not in the form CLUSTER/ROLE/ENV/NAME/INSTANCE' in exc.value.message)
+ assert mock_subprocess.call_count == 0
+
+ def test_scp_bad_jobkey_invalid_format(self):
+ """Test the scp command fails when given general scp format."""
+ self._mock_api.query.return_value = self.create_status_response()
+ self._mock_options.source = ['root@192.168.0.1:test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/1:test/dir'
+
+ with contextlib.nested(patch('subprocess.call', return_value=EXIT_OK)) as [mock_subprocess]:
+ with pytest.raises(Context.CommandError) as exc:
+ assert self._command.execute(self._fake_context) == EXIT_INVALID_PARAMETER
+ assert('not in the form CLUSTER/ROLE/ENV/NAME/INSTANCE' in exc.value.message)
+ assert mock_subprocess.call_count == 0
+
+ def test_scp_job_not_found(self):
+ """Test the scp command when the jobkey parameter specifies a job that isn't running."""
+ self._mock_api.query.return_value = self.create_nojob_status_response()
+ self._mock_options.source = ['test.txt']
+ self._mock_options.dest = 'west/bozo/test/hello/0:test/dir'
+
+ with contextlib.nested(patch('subprocess.call', return_value=EXIT_OK)) as [mock_subprocess]:
+ with pytest.raises(Context.CommandError) as exc:
+ assert self._command.execute(self._fake_context) == EXIT_INVALID_PARAMETER
+ assert(ScpCommand.JOB_NOT_FOUND_ERROR_MSG % ('west/bozo/test/hello', '0')
+ in exc.value.message)
+ assert mock_subprocess.call_count == 0
http://git-wip-us.apache.org/repos/asf/aurora/blob/4c0974bc/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
index 1a81dc5..f0819fb 100755
--- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
+++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
@@ -351,6 +351,55 @@ test_run() {
[[ "$sandbox_contents" = " 3 .logs" ]]
}
+test_scp_success() {
+ local _jobkey=$1/0
+ local _filename=scp_success.txt
+ local _expected_return=" 1 scp_success.txt"
+
+ # Unset because grep can return 1 if the file does not exist
+ set +e
+
+ # Ensure file does not exists before scp
+ pre_sandbox_contents=$(aurora task run $_jobkey "ls" | awk '{print $2}' | grep ${_filename} | sort | uniq -c)
+ [[ "$pre_sandbox_contents" != $_expected_return ]]
+
+ # Reset -e after command has been run
+ set -e
+
+ # Create a file and move it to the sandbox of a job
+ touch $_filename
+ aurora task scp $_filename ${_jobkey}:
+ sandbox_contents=$(aurora task run $_jobkey "ls" | awk '{print $2}' | grep ${_filename} | sort | uniq -c)
+ [[ "$sandbox_contents" == $_expected_return ]]
+}
+
+test_scp_permissions() {
+ local _jobkey=$1/0
+ local _filename=scp_fail_permission.txt
+ local _retcode=0
+ local _sandbox_contents
+ # Create a file and try to move it, ensure we get permission denied
+ touch $_filename
+
+ # Unset because we are expecting an error
+ set +e
+
+ _sandbox_contents=$(aurora task scp $_filename ${_jobkey}:../ 2>&1 > /dev/null)
+ _retcode=$?
+
+ # Reset -e after command has been run
+ set -e
+
+ if [[ "$_retcode" != 1 ]]; then
+ echo "Permission to exit chroot jail given when should have failed"
+ exit 1
+ fi
+ if [[ "$_sandbox_contents" != *"../scp_fail_permission.txt: Permission denied"* ]]; then
+ echo "Unexpected response from invalid scp command"
+ exit 1
+ fi
+}
+
test_kill() {
local _jobkey=$1
shift 1
@@ -440,6 +489,13 @@ test_http_example() {
test_update $_jobkey $_updated_config $_cluster $_bind_parameters
test_announce $_role $_env $_job
test_run $_jobkey
+ # TODO(AURORA-1926): 'aurora task scp' only works fully on Mesos containers (can only read for
+ # Docker). See if it is possible to enable write for Docker sandboxes as well then remove the
+ # 'if' guard below.
+ if [[ $_job != *"docker"* ]]; then
+ test_scp_success $_jobkey
+ test_scp_permissions $_jobkey
+ fi
test_kill $_jobkey
test_quota $_cluster $_role
}