You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2016/06/28 06:42:57 UTC

[1/2] mesos git commit: Added helper function for /containers endpoint.

Repository: mesos
Updated Branches:
  refs/heads/master 37dc886ee -> f9fe7f47c


Added helper function for /containers endpoint.

The helper function returns JSON::Array, which can be
reused by v1 Operator API GET_CONTAINERS.

Review: https://reviews.apache.org/r/49269/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c34d55c0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c34d55c0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c34d55c0

Branch: refs/heads/master
Commit: c34d55c098737994d45482ea10cf38443d563bfe
Parents: 37dc886
Author: Jay Guo <gu...@cn.ibm.com>
Authored: Mon Jun 27 23:41:15 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Jun 27 23:41:15 2016 -0700

----------------------------------------------------------------------
 src/slave/http.cpp  | 33 ++++++++++++++++++++++++---------
 src/slave/slave.hpp |  3 +++
 2 files changed, 27 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/c34d55c0/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index 30006c2..4917416 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -1148,6 +1148,27 @@ Future<Response> Slave::Http::containers(
 
 Future<Response> Slave::Http::_containers(const Request& request) const
 {
+  return __containers()
+      .then([request](const Future<JSON::Array>& result) -> Future<Response> {
+        if (!result.isReady()) {
+          LOG(WARNING) << "Could not collect container status and statistics: "
+                       << (result.isFailed()
+                            ? result.failure()
+                            : "Discarded");
+
+          return result.isFailed()
+            ? InternalServerError(result.failure())
+            : InternalServerError();
+        }
+
+        return process::http::OK(
+            result.get(), request.url.query.get("jsonp"));
+      });
+}
+
+
+Future<JSON::Array> Slave::Http::__containers() const
+{
   Owned<list<JSON::Object>> metadata(new list<JSON::Object>());
   list<Future<ContainerStatus>> statusFutures;
   list<Future<ResourceStatistics>> statsFutures;
@@ -1171,10 +1192,10 @@ Future<Response> Slave::Http::_containers(const Request& request) const
   }
 
   return await(await(statusFutures), await(statsFutures)).then(
-      [metadata, request](const tuple<
+      [metadata](const tuple<
           Future<list<Future<ContainerStatus>>>,
           Future<list<Future<ResourceStatistics>>>>& t)
-          -> Future<Response> {
+          -> Future<JSON::Array> {
         const list<Future<ContainerStatus>>& status = std::get<0>(t).get();
         const list<Future<ResourceStatistics>>& stats = std::get<1>(t).get();
         CHECK_EQ(status.size(), stats.size());
@@ -1222,13 +1243,7 @@ Future<Response> Slave::Http::_containers(const Request& request) const
           metadataIter++;
         }
 
-        return process::http::OK(result, request.url.query.get("jsonp"));
-      })
-      .repair([](const Future<Response>& future) {
-        LOG(WARNING) << "Could not collect container status and statistics: "
-                     << future.failure();
-
-        return InternalServerError();
+        return result;
       });
 }
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/c34d55c0/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 58ff2bf..383fb08 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -496,6 +496,9 @@ private:
     process::Future<process::http::Response> _containers(
         const process::http::Request& request) const;
 
+    // Helper function to collect containers status and resource statistics.
+    process::Future<JSON::Array> __containers() const;
+
     // Helper routines for endpoint authorization.
     Try<std::string> extractEndpoint(const process::http::URL& url) const;
 


[2/2] mesos git commit: Implemented GET_CONTAINERS Call in v1 Agent API.

Posted by vi...@apache.org.
Implemented GET_CONTAINERS Call in v1 Agent API.

Review: https://reviews.apache.org/r/49015/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f9fe7f47
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f9fe7f47
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f9fe7f47

Branch: refs/heads/master
Commit: f9fe7f47c007b7d53ae297d615701d9130117762
Parents: c34d55c
Author: Jay Guo <gu...@cn.ibm.com>
Authored: Mon Jun 27 23:41:21 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Jun 27 23:41:21 2016 -0700

----------------------------------------------------------------------
 include/mesos/agent/agent.proto    |  10 ++-
 include/mesos/v1/agent/agent.proto |  10 ++-
 src/internal/evolve.cpp            |  57 +++++++++++++++++
 src/internal/evolve.hpp            |   3 +
 src/slave/http.cpp                 |  31 +++++++++-
 src/slave/slave.hpp                |   5 ++
 src/tests/api_tests.cpp            | 105 ++++++++++++++++++++++++++++++++
 7 files changed, 218 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/include/mesos/agent/agent.proto
----------------------------------------------------------------------
diff --git a/include/mesos/agent/agent.proto b/include/mesos/agent/agent.proto
index 81eede9..538d12f 100644
--- a/include/mesos/agent/agent.proto
+++ b/include/mesos/agent/agent.proto
@@ -152,9 +152,17 @@ message Response {
     // TODO(vinod): Fill in the fields.
   }
 
+  // Information about containers running on this agent. It contains
+  // ContainerStatus and ResourceStatistics along with some metadata
+  // of the containers.
   message GetContainers {
     message Container {
-      // TODO(vinod): Fill in the fields.
+      required FrameworkID framework_id = 1;
+      required ExecutorID executor_id = 2;
+      required string executor_name = 3;
+      required ContainerID container_id = 4;
+      optional ContainerStatus container_status = 5;
+      optional ResourceStatistics resource_statistics = 6;
     }
 
     repeated Container containers = 1;

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/include/mesos/v1/agent/agent.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/agent/agent.proto b/include/mesos/v1/agent/agent.proto
index 52655c6..48f1517 100644
--- a/include/mesos/v1/agent/agent.proto
+++ b/include/mesos/v1/agent/agent.proto
@@ -152,9 +152,17 @@ message Response {
     // TODO(vinod): Fill in the fields.
   }
 
+  // Information about containers running on this agent. It contains
+  // ContainerStatus and ResourceStatistics along with some metadata
+  // of the containers.
   message GetContainers {
     message Container {
-      // TODO(vinod): Fill in the fields.
+      required FrameworkID framework_id = 1;
+      required ExecutorID executor_id = 2;
+      required string executor_name = 3;
+      required ContainerID container_id = 4;
+      optional ContainerStatus container_status = 5;
+      optional ResourceStatistics resource_statistics = 6;
     }
 
     repeated Container containers = 1;

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/src/internal/evolve.cpp
----------------------------------------------------------------------
diff --git a/src/internal/evolve.cpp b/src/internal/evolve.cpp
index f63d606..bb99cb2 100644
--- a/src/internal/evolve.cpp
+++ b/src/internal/evolve.cpp
@@ -593,5 +593,62 @@ v1::agent::Response evolve<v1::agent::Response::GET_VERSION>(
   return response;
 }
 
+
+template<>
+v1::agent::Response evolve<v1::agent::Response::GET_CONTAINERS>(
+    const JSON::Array& array)
+{
+  v1::agent::Response response;
+  response.set_type(v1::agent::Response::GET_CONTAINERS);
+
+  foreach (const JSON::Value& value, array.values) {
+    v1::agent::Response::GetContainers::Container* container =
+      response.mutable_get_containers()->add_containers();
+
+    JSON::Object object = value.as<JSON::Object>();
+
+    Result<JSON::String> container_id =
+      object.find<JSON::String>("container_id");
+    CHECK_SOME(container_id);
+    container->mutable_container_id()->set_value(container_id.get().value);
+
+    Result<JSON::String> framework_id =
+      object.find<JSON::String>("framework_id");
+    CHECK_SOME(framework_id);
+    container->mutable_framework_id()->set_value(framework_id.get().value);
+
+    Result<JSON::String> executor_id =
+      object.find<JSON::String>("executor_id");
+    CHECK_SOME(executor_id);
+    container->mutable_executor_id()->set_value(executor_id.get().value);
+
+    Result<JSON::String> executor_name =
+      object.find<JSON::String>("executor_name");
+    CHECK_SOME(executor_name);
+    container->set_executor_name(executor_name.get().value);
+
+    Result<JSON::Object> container_status =
+      object.find<JSON::Object>("status");
+    if (container_status.isSome()) {
+      Try<mesos::v1::ContainerStatus> status =
+        protobuf::parse<mesos::v1::ContainerStatus>(container_status.get());
+      CHECK_SOME(status);
+      container->mutable_container_status()->CopyFrom(status.get());
+    }
+
+    Result<JSON::Object> resource_statistics =
+      object.find<JSON::Object>("statistics");
+    if (resource_statistics.isSome()) {
+      Try<mesos::v1::ResourceStatistics> statistics =
+        protobuf::parse<mesos::v1::ResourceStatistics>(
+            resource_statistics.get());
+      CHECK_SOME(statistics);
+      container->mutable_resource_statistics()->CopyFrom(statistics.get());
+    }
+  }
+
+  return response;
+}
+
 } // namespace internal {
 } // namespace mesos {

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/src/internal/evolve.hpp
----------------------------------------------------------------------
diff --git a/src/internal/evolve.hpp b/src/internal/evolve.hpp
index a8648f5..582f347 100644
--- a/src/internal/evolve.hpp
+++ b/src/internal/evolve.hpp
@@ -151,6 +151,9 @@ v1::master::Response evolve(const mesos::maintenance::Schedule& schedule);
 template <v1::agent::Response::Type T>
 v1::agent::Response evolve(const JSON::Object& object);
 
+template <v1::agent::Response::Type T>
+v1::agent::Response evolve(const JSON::Array& array);
+
 } // namespace internal {
 } // namespace mesos {
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index 4917416..f325bc6 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -387,7 +387,7 @@ Future<Response> Slave::Http::api(
       return NotImplemented();
 
     case agent::Call::GET_CONTAINERS:
-      return NotImplemented();
+      return getContainers(call, principal, acceptType);
   }
 
   UNREACHABLE();
@@ -1146,6 +1146,35 @@ Future<Response> Slave::Http::containers(
 }
 
 
+Future<Response> Slave::Http::getContainers(
+    const agent::Call& call,
+    const Option<string>& printcipal,
+    ContentType contentType) const
+{
+  CHECK_EQ(agent::Call::GET_CONTAINERS, call.type());
+
+  return __containers()
+      .then([contentType](const Future<JSON::Array>& result)
+          -> Future<Response> {
+        if (!result.isReady()) {
+          LOG(WARNING) << "Could not collect container status and statistics: "
+                       << (result.isFailed()
+                            ? result.failure()
+                            : "Discarded");
+          return result.isFailed()
+            ? InternalServerError(result.failure())
+            : InternalServerError();
+        }
+
+        return OK(
+            serialize(
+                contentType,
+                evolve<v1::agent::Response::GET_CONTAINERS>(result.get())),
+            stringify(contentType));
+      });
+}
+
+
 Future<Response> Slave::Http::_containers(const Request& request) const
 {
   return __containers()

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 383fb08..2afd7d1 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -546,6 +546,11 @@ private:
         const Option<std::string>& principal,
         ContentType contentType) const;
 
+    process::Future<process::http::Response> getContainers(
+        const mesos::agent::Call& call,
+        const Option<std::string>& principal,
+        ContentType contentType) const;
+
     Slave* slave;
 
     // Used to rate limit the statistics endpoint.

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9fe7f47/src/tests/api_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/api_tests.cpp b/src/tests/api_tests.cpp
index 7eb179f..c8f3a7d 100644
--- a/src/tests/api_tests.cpp
+++ b/src/tests/api_tests.cpp
@@ -1579,6 +1579,111 @@ TEST_P(AgentAPITest, SetLoggingLevel)
   Clock::resume();
 }
 
+
+TEST_P(AgentAPITest, GetContainers)
+{
+  Try<Owned<cluster::Master>> master = StartMaster();
+  ASSERT_SOME(master);
+
+  MockExecutor exec(DEFAULT_EXECUTOR_ID);
+  TestContainerizer containerizer(&exec);
+  StandaloneMasterDetector detector(master.get()->pid);
+
+  Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer);
+  ASSERT_SOME(slave);
+
+  MockScheduler sched;
+  MesosSchedulerDriver driver(
+      &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+  EXPECT_CALL(sched, registered(_, _, _));
+  EXPECT_CALL(exec, registered(_, _, _, _));
+
+  Future<vector<Offer>> offers;
+  EXPECT_CALL(sched, resourceOffers(&driver, _))
+    .WillOnce(FutureArg<1>(&offers))
+    .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+  driver.start();
+
+  AWAIT_READY(offers);
+  EXPECT_NE(0u, offers.get().size());
+
+  const Offer& offer = offers.get()[0];
+
+  TaskInfo task = createTask(
+      offer.slave_id(),
+      Resources::parse("cpus:0.1;mem:32").get(),
+      "sleep 1000",
+      exec.id);
+
+  EXPECT_CALL(exec, launchTask(_, _))
+    .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
+
+  Future<TaskStatus> status;
+  EXPECT_CALL(sched, statusUpdate(&driver, _))
+    .WillOnce(FutureArg<1>(&status));
+
+  // No tasks launched, we should expect zero containers in Response.
+  {
+    v1::agent::Call v1Call;
+    v1Call.set_type(v1::agent::Call::GET_CONTAINERS);
+
+    ContentType contentType = GetParam();
+
+    Future<v1::agent::Response> v1Response =
+      post(slave.get()->pid, v1Call, contentType);
+
+    AWAIT_READY(v1Response);
+    ASSERT_TRUE(v1Response.get().IsInitialized());
+    ASSERT_EQ(v1::agent::Response::GET_CONTAINERS, v1Response.get().type());
+    ASSERT_EQ(0, v1Response.get().get_containers().containers_size());
+  }
+
+  driver.launchTasks(offer.id(), {task});
+
+  AWAIT_READY(status);
+  EXPECT_EQ(TASK_RUNNING, status.get().state());
+
+  ResourceStatistics statistics;
+  statistics.set_mem_limit_bytes(2048);
+  // We have to set timestamp here since serializing protobuf without
+  // filling all required fields generates errors.
+  statistics.set_timestamp(0);
+
+  EXPECT_CALL(containerizer, usage(_))
+    .WillOnce(Return(statistics));
+
+  ContainerStatus containerStatus;
+  NetworkInfo* networkInfo = containerStatus.add_network_infos();
+  NetworkInfo::IPAddress* ipAddr = networkInfo->add_ip_addresses();
+  ipAddr->set_ip_address("192.168.1.20");
+
+  EXPECT_CALL(containerizer, status(_))
+    .WillOnce(Return(containerStatus));
+
+  v1::agent::Call v1Call;
+  v1Call.set_type(v1::agent::Call::GET_CONTAINERS);
+
+  ContentType contentType = GetParam();
+  Future<v1::agent::Response> v1Response =
+    post(slave.get()->pid, v1Call, contentType);
+
+  AWAIT_READY(v1Response);
+  ASSERT_TRUE(v1Response.get().IsInitialized());
+  ASSERT_EQ(v1::agent::Response::GET_CONTAINERS, v1Response.get().type());
+  ASSERT_EQ(1, v1Response.get().get_containers().containers_size());
+  ASSERT_EQ("192.168.1.20",
+            v1Response.get().get_containers().containers(0).container_status()
+              .network_infos(0).ip_addresses(0).ip_address());
+
+  EXPECT_CALL(exec, shutdown(_))
+    .Times(AtMost(1));
+
+  driver.stop();
+  driver.join();
+}
+
 } // namespace tests {
 } // namespace internal {
 } // namespace mesos {