You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@mesos.apache.org by Zhitao Li <zh...@gmail.com> on 2017/09/13 01:25:58 UTC
Review Request 62263: Add test to browse and read in sandbox virtual
path.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/62263/
-----------------------------------------------------------
Review request for mesos, Benjamin Mahler and Jason Lai.
Bugs: MESOS-7899
https://issues.apache.org/jira/browse/MESOS-7899
Repository: mesos
Description
-------
Add test to browse and read in sandbox virtual path.
Diffs
-----
src/tests/slave_tests.cpp 1bdadce4c50cbff958f2be2a4261e130b414acfd
Diff: https://reviews.apache.org/r/62263/diff/1/
Testing
-------
Thanks,
Zhitao Li
Re: Review Request 62263: Add test to browse and read in sandbox
virtual path.
Posted by Benjamin Mahler <bm...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/62263/#review185738
-----------------------------------------------------------
Fix it, then Ship it!
Made the tweaks based on the comments to avoid another round of reviewing here, see the diff below:
```
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index 4428a99d8..949e2d44d 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -7800,8 +7800,12 @@ TEST_F(SlaveTest, IgnoreV0ExecutorIfItReregistersWithoutReconnect)
// This test verifies that an executor's latest run directory can
-// be browsed by virtual path.
-TEST_F(SlaveTest, BrowseSandboxByVirtualpath)
+// be browsed via the `/files` endpoint both while the executor is
+// still running and after the executor terminates.
+//
+// Note that we only test the recommended virtual path format:
+// `/framework/FID/executor/EID/latest`.
+TEST_F(SlaveTest, BrowseExecutorSandboxByVirtualPath)
{
master::Flags masterFlags = this->CreateMasterFlags();
Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
@@ -7809,10 +7813,6 @@ TEST_F(SlaveTest, BrowseSandboxByVirtualpath)
slave::Flags agentFlags = this->CreateSlaveFlags();
- agentFlags.hostname = "localhost";
- agentFlags.resources = "cpus:4;gpus:0;mem:2048;disk:512;ports:[33000-34000]";
- agentFlags.attributes = "rack:abc;host:myhost";
-
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
@@ -7847,8 +7847,6 @@ TEST_F(SlaveTest, BrowseSandboxByVirtualpath)
AWAIT_READY(offers);
EXPECT_FALSE(offers->empty());
- FrameworkID frameworkId = offers->front().framework_id();
- SlaveID slaveId = offers->front().slave_id();
Resources executorResources = Resources::parse("cpus:0.1;mem:32").get();
executorResources.allocate("*");
@@ -7880,6 +7878,10 @@ TEST_F(SlaveTest, BrowseSandboxByVirtualpath)
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status->state());
+ // Manually inject a file into the sandbox.
+ FrameworkID frameworkId = offers->front().framework_id();
+ SlaveID slaveId = offers->front().slave_id();
+
const string latestRunPath = paths::getExecutorLatestRunPath(
agentFlags.work_dir, slaveId, frameworkId, DEFAULT_EXECUTOR_ID);
EXPECT_TRUE(os::exists(latestRunPath));
@@ -7898,8 +7900,8 @@ TEST_F(SlaveTest, BrowseSandboxByVirtualpath)
query,
createBasicAuthHeaders(DEFAULT_CREDENTIAL));
- AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
- AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
+ AWAIT_ASSERT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_ASSERT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
Try<JSON::Array> parse = JSON::parse<JSON::Array>(response->body);
ASSERT_SOME(parse);
@@ -7916,7 +7918,58 @@ TEST_F(SlaveTest, BrowseSandboxByVirtualpath)
query,
createBasicAuthHeaders(DEFAULT_CREDENTIAL));
- AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_ASSERT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ JSON::Object expected;
+ expected.values["offset"] = 0;
+ expected.values["data"] = "testing";
+
+ AWAIT_EXPECT_RESPONSE_BODY_EQ(stringify(expected), response);
+ }
+
+ // Now destroy the executor and make sure that the sandbox is
+ // still available. We're sure that the GC won't prune the
+ // sandbox since the clock is paused.
+ Future<TaskStatus> status2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status2));
+
+ EXPECT_CALL(sched, executorLost(_, _, _, _))
+ .Times(AtMost(1));
+
+ AWAIT_READY(containerizer.destroy(frameworkId, DEFAULT_EXECUTOR_ID));
+ //AWAIT_READY(shutdown);
+
+ AWAIT_READY(status2);
+ EXPECT_EQ(TASK_FAILED, status2->state());
+
+ {
+ const string query = string("path=") + virtualPath;
+ Future<Response> response = process::http::get(
+ files,
+ "browse",
+ query,
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_ASSERT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_ASSERT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
+
+ Try<JSON::Array> parse = JSON::parse<JSON::Array>(response->body);
+ ASSERT_SOME(parse);
+ EXPECT_NE(0, parse->values.size());
+ }
+
+ {
+ const string query =
+ string("path=") + path::join(virtualPath, "foo.bar") + "&offset=0";
+ process::UPID files("files", slave.get()->pid.address);
+ Future<Response> response = process::http::get(
+ files,
+ "read",
+ query,
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_ASSERT_RESPONSE_STATUS_EQ(OK().status, response);
JSON::Object expected;
expected.values["offset"] = 0;
```
src/tests/slave_tests.cpp
Lines 7804 (patched)
<https://reviews.apache.org/r/62263/#comment262076>
s/Sandbox/ExecutorSandbox/
s/p/P/
src/tests/slave_tests.cpp
Lines 7812-7814 (patched)
<https://reviews.apache.org/r/62263/#comment262077>
Why did you need this?
src/tests/slave_tests.cpp
Lines 7893-7926 (patched)
<https://reviews.apache.org/r/62263/#comment262082>
It would be good to also test that these are accessible after the executor terminates.
- Benjamin Mahler
On Sept. 13, 2017, 1:25 a.m., Zhitao Li wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/62263/
> -----------------------------------------------------------
>
> (Updated Sept. 13, 2017, 1:25 a.m.)
>
>
> Review request for mesos, Benjamin Mahler and Jason Lai.
>
>
> Bugs: MESOS-7899
> https://issues.apache.org/jira/browse/MESOS-7899
>
>
> Repository: mesos
>
>
> Description
> -------
>
> Add test to browse and read in sandbox virtual path.
>
>
> Diffs
> -----
>
> src/tests/slave_tests.cpp 1bdadce4c50cbff958f2be2a4261e130b414acfd
>
>
> Diff: https://reviews.apache.org/r/62263/diff/1/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Zhitao Li
>
>
Re: Review Request 62263: Add test to browse and read in sandbox
virtual path.
Posted by Mesos Reviewbot Windows <re...@mesos.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/62263/#review185262
-----------------------------------------------------------
FAIL: Some Mesos tests failed.
Reviews applied: `['62040', '62174', '62047', '62263']`
Failed command: `C:\mesos\src\mesos-tests.exe --verbose`
All the build artifacts available at: http://dcos-win.westus.cloudapp.azure.com/mesos-build/review/62263
Relevant logs:
- [mesos-tests-stdout.log](http://dcos-win.westus.cloudapp.azure.com/mesos-build/review/62263/logs/mesos-tests-stdout.log):
```
[ OK ] ContentType/SchedulerTest.SchedulerReconnect/0 (247 ms)
[ RUN ] ContentType/SchedulerTest.SchedulerReconnect/1
[ OK ] ContentType/SchedulerTest.SchedulerReconnect/1 (257 ms)
[----------] 30 tests from ContentType/SchedulerTest (24722 ms total)
[----------] 2 tests from ContentTypeAndSSLConfig/SchedulerSSLTest
[ RUN ] ContentTypeAndSSLConfig/SchedulerSSLTest.RunTaskAndTeardown/0
[ OK ] ContentTypeAndSSLConfig/SchedulerSSLTest.RunTaskAndTeardown/0 (900 ms)
[ RUN ] ContentTypeAndSSLConfig/SchedulerSSLTest.RunTaskAndTeardown/1
[ OK ] ContentTypeAndSSLConfig/SchedulerSSLTest.RunTaskAndTeardown/1 (1023 ms)
[----------] 2 tests from ContentTypeAndSSLConfig/SchedulerSSLTest (1995 ms total)
[----------] 2 tests from ContainerizerType/DefaultContainerDNSFlagTest
[ RUN ] ContainerizerType/DefaultContainerDNSFlagTest.ValidateFlag/0
[ OK ] ContainerizerType/DefaultContainerDNSFlagTest.ValidateFlag/0 (141 ms)
[ RUN ] ContainerizerType/DefaultContainerDNSFlagTest.ValidateFlag/1
[ OK ] ContainerizerType/DefaultContainerDNSFlagTest.ValidateFlag/1 (153 ms)
[----------] 2 tests from ContainerizerType/DefaultContainerDNSFlagTest (344 ms total)
[----------] Global test environment tear-down
[==========] 628 tests from 66 test cases ran. (340272 ms total)
[ PASSED ] 626 tests.
[ FAILED ] 2 tests, listed below:
[ FAILED ] ContentType/MasterAPITest.EventAuthorizationFiltering/1, where GetParam() = application/json
[ FAILED ] MesosContainerizer/DefaultExecutorTest.SigkillExecutor/0, where GetParam() = "mesos"
2 FAILED TESTS
YOU HAVE 174 DISABLED TESTS
```
- [mesos-tests-stderr.log](http://dcos-win.westus.cloudapp.azure.com/mesos-build/review/62263/logs/mesos-tests-stderr.log):
```
I0913 05:23:24.409837 10332 master.cpp:8418] Removing framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 (default)
I0913 05:23:24.409837 10332 master.cpp:3267] Deactivating framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 (default)
I0913 05:23:24.409837 14024 hierarchical.cpp:412] Deactivated framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000
I0913 05:23:24.410820 13384 slave.cpp:3245] Shutting down framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000
I0913 05:23:24.410820 10332 master.cpp:8993] Updating the state of task ac5711a8-999c-41be-9d60-7a4fb942fe07 of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 (latest state: TASK_KILLED, status update state: TASK_KILLED)
I0913 05:23:24.410820 13384 slave.cpp:5752] Shutting down executor 'default' of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 (via HTTP)
I0913 05:23:24.418820 10332 master.cpp:9087] Removing task ac5711a8-999c-41be-9d60-7a4fb942fe07 with resources [{"allocation_info":{"role":"*"},"name":"cpus","scalar":{"value":2.0},"type":"SCALAR"},{"allocation_info":{"role":"*"},"name":"mem","scalar":{"value":1024.0},"type":"SCALAR"},{"allocation_info":{"role":"*"},"name":"disk","scalar":{"value":1024.0},"type":"SCALAR"},{"allocation_info":{"role":"*"},"name":"ports","ranges":{"range":[{"begin":31000,"end":32000}]},"type":"RANGES"}] of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 on agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 at slave(255)@10.3.1.7:58816 (mesos-bld-s2.zq4gs31qjdiunm1ryi1452nvnh.dx.internal.cloudapp.net)
I0913 05:23:24.438820 10332 master.cpp:9116] Removing executor 'default' with resources [] of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 on agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 at slave(255)@10.3.1.7:58816 (mesos-bld-s2.zq4gs31qjdiunm1ryi1452nvnh.dx.internal.cloudapp.net)
I0913 05:23:24.442886 13772 hierarchical.cpp:355] Removed framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000
E0913 05:23:24.443821 10920 scheduler.cpp:649] End-Of-File received from master. The master closed the event stream
I0913 05:23:24.453820 10920 scheduler.cpp:444] Re-detecting master
I0913 05:23:24.456822 10920 scheduler.cpp:470] New master detected at master@10.3.1.7:58816
I0913 05:23:24.463822 11232 slave.cpp:5417] Executor 'default' of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 exited with status 0
I0913 05:23:24.463822 11232 slave.cpp:5521] Cleaning up executor 'default' of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 (via HTTP)
W0913 05:23:24.463822 13772 master.cpp:7021] Ignoring unknown exited executor 'default' of framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000 on agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 at slave(255)@10.3.1.7:58816 (mesos-bld-s2.zq4gs31qjdiunm1ryi1452nvnh.dx.internal.cloudapp.net)
I0913 05:23:24.469823 4736 gc.cpp:91] Scheduling 'C:\Users\mesos\AppData\Local\Temp\2\0HSPbt\slaves\a1994e31-61ae-4465-9f33-a8df4e1213df-S0\frameworks\a1994e31-61ae-4465-9f33-a8df4e1213df-0000\executors\default\runs\e9475aab-106d-4fa2-9e71-528ab18bf680' for gc 6.99999457381926days in the future
I0913 05:23:24.471823 11232 slave.cpp:5628] Cleaning up framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000
I0913 05:23:24.478824 14024 status_update_manager.cpp:285] Closing status update streams for framework a1994e31-61ae-4465-9f33-a8df4e1213df-0000
I0913 05:23:24.479822 4736 gc.cpp:91] Scheduling 'C:\Users\mesos\AppData\Local\Temp\2\0HSPbt\slaves\a1994e31-61ae-4465-9f33-a8df4e1213df-S0\frameworks\a1994e31-61ae-4465-9f33-a8df4e1213df-0000\executors\default' for gc 6.99999455067259days in the future
I0913 05:23:24.479822 11232 slave.cpp:866] Agent terminating
I0913 05:23:24.479822 4736 gc.cpp:91] Scheduling 'C:\Users\mesos\AppData\Local\Temp\2\0HSPbt\slaves\a1994e31-61ae-4465-9f33-a8df4e1213df-S0\frameworks\a1994e31-61ae-4465-9f33-a8df4e1213df-0000' for gc 6.99999444649185days in the future
I0913 05:23:24.480823 10488 master.cpp:1321] Agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 at slave(255)@10.3.1.7:58816 (mesos-bld-s2.zq4gs31qjdiunm1ryi1452nvnh.dx.internal.cloudapp.net) disconnected
I0913 05:23:24.491823 10488 master.cpp:3304] Disconnecting agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 at slave(255)@10.3.1.7:58816 (mesos-bld-s2.zq4gs31qjdiunm1ryi1452nvnh.dx.internal.cloudapp.net)
I0913 05:23:24.492822 10488 master.cpp:3323] Deactivating agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 at slave(255)@10.3.1.7:58816 (mesos-bld-s2.zq4gs31qjdiunm1ryi1452nvnh.dx.internal.cloudapp.net)
I0913 05:23:24.492822 13772 hierarchical.cpp:690] Agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0 deactivated
I0913 05:23:24.507823 10920 master.cpp:1163] Master terminating
I0913 05:23:24.512823 11232 hierarchical.cpp:626] Removed agent a1994e31-61ae-4465-9f33-a8df4e1213df-S0
W0913 05:23:24.521824 13740 master.hpp:2761] Failed to close HTTP pipe for a1994e31-61ae-4465-9f33-a8df4e1213df-0000 (default)
I0913 05:23:25.176877 13992 process.cpp:1068] Failed to accept socket: future discarded
```
- Mesos Reviewbot Windows
On Sept. 13, 2017, 1:25 a.m., Zhitao Li wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/62263/
> -----------------------------------------------------------
>
> (Updated Sept. 13, 2017, 1:25 a.m.)
>
>
> Review request for mesos, Benjamin Mahler and Jason Lai.
>
>
> Bugs: MESOS-7899
> https://issues.apache.org/jira/browse/MESOS-7899
>
>
> Repository: mesos
>
>
> Description
> -------
>
> Add test to browse and read in sandbox virtual path.
>
>
> Diffs
> -----
>
> src/tests/slave_tests.cpp 1bdadce4c50cbff958f2be2a4261e130b414acfd
>
>
> Diff: https://reviews.apache.org/r/62263/diff/1/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Zhitao Li
>
>