You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2016/08/02 19:25:12 UTC
[1/3] mesos git commit: Libprocess: Fixed decoder to support
incremental URL parsing.
Repository: mesos
Updated Branches:
refs/heads/master 2eeb98e73 -> d566536f6
Libprocess: Fixed decoder to support incremental URL parsing.
Review: https://reviews.apache.org/r/50634
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2776a09c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2776a09c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2776a09c
Branch: refs/heads/master
Commit: 2776a09cbcd836080241a5ad8c1e003984e5a146
Parents: 2eeb98e
Author: Joris Van Remoortere <jo...@gmail.com>
Authored: Sat Jul 30 12:58:28 2016 -0700
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Tue Aug 2 11:31:02 2016 -0700
----------------------------------------------------------------------
3rdparty/libprocess/src/decoder.hpp | 70 +++++++++++++++++++-------------
1 file changed, 41 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/2776a09c/3rdparty/libprocess/src/decoder.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/decoder.hpp b/3rdparty/libprocess/src/decoder.hpp
index c0cefd4..47cfc0f 100644
--- a/3rdparty/libprocess/src/decoder.hpp
+++ b/3rdparty/libprocess/src/decoder.hpp
@@ -101,6 +101,7 @@ private:
decoder->field.clear();
decoder->value.clear();
decoder->query.clear();
+ decoder->url.clear();
CHECK(decoder->request == nullptr);
@@ -154,38 +155,17 @@ private:
{
DataDecoder* decoder = (DataDecoder*) p->data;
CHECK_NOTNULL(decoder->request);
- int result = 0;
#if (HTTP_PARSER_VERSION_MAJOR >= 2)
// Reworked parsing for version >= 2.0.
- http_parser_url url;
- result = http_parser_parse_url(data, length, 0, &url);
-
- if (result == 0) {
- if (url.field_set & (1 << UF_PATH)) {
- decoder->request->url.path.append(
- data + url.field_data[UF_PATH].off,
- url.field_data[UF_PATH].len);
- }
-
- if (url.field_set & (1 << UF_FRAGMENT)) {
- if (decoder->request->url.fragment.isNone()) {
- decoder->request->url.fragment = "";
- }
- decoder->request->url.fragment->append(
- data + url.field_data[UF_FRAGMENT].off,
- url.field_data[UF_FRAGMENT].len);
- }
-
- if (url.field_set & (1 << UF_QUERY)) {
- decoder->query.append(
- data + url.field_data[UF_QUERY].off,
- url.field_data[UF_QUERY].len);
- }
- }
+ // The current http_parser library (version 2.6.2 and below)
+ // does not support incremental parsing of URLs. To compensate
+ // we incrementally collect the data and parse it in
+ // `on_message_complete`.
+ decoder->url.append(data, length);
#endif
- return result;
+ return 0;
}
static int on_header_field(http_parser* p, const char* data, size_t length)
@@ -245,6 +225,39 @@ private:
{
DataDecoder* decoder = (DataDecoder*) p->data;
+ CHECK_NOTNULL(decoder->request);
+
+#if (HTTP_PARSER_VERSION_MAJOR >= 2)
+ // Parse the URL. This data was incrementally built up during calls
+ // to `on_url`.
+ http_parser_url url;
+ http_parser_url_init(&url);
+ int parse_url =
+ http_parser_parse_url(decoder->url.data(), decoder->url.size(), 0, &url);
+
+ if (parse_url != 0) {
+ return parse_url;
+ }
+
+ if (url.field_set & (1 << UF_PATH)) {
+ decoder->request->url.path = std::string(
+ decoder->url.data() + url.field_data[UF_PATH].off,
+ url.field_data[UF_PATH].len);
+ }
+
+ if (url.field_set & (1 << UF_FRAGMENT)) {
+ decoder->request->url.fragment = std::string(
+ decoder->url.data() + url.field_data[UF_FRAGMENT].off,
+ url.field_data[UF_FRAGMENT].len);
+ }
+
+ if (url.field_set & (1 << UF_QUERY)) {
+ decoder->query = std::string(
+ decoder->url.data() + url.field_data[UF_QUERY].off,
+ url.field_data[UF_QUERY].len);
+ }
+#endif
+
// Parse the query key/values.
Try<hashmap<std::string, std::string>> decoded =
http::query::decode(decoder->query);
@@ -253,8 +266,6 @@ private:
return 1;
}
- CHECK_NOTNULL(decoder->request);
-
decoder->request->url.query = decoded.get();
Option<std::string> encoding =
@@ -291,6 +302,7 @@ private:
std::string field;
std::string value;
std::string query;
+ std::string url;
http::Request* request;
[3/3] mesos git commit: Removed `O_SYNC` from StatusUpdateManager.
Posted by jo...@apache.org.
Removed `O_SYNC` from StatusUpdateManager.
The StatusUpdateManager data is not recovered after reboot. This means
we can rely on flushes to the page cache to be sufficient for writing
status updates.
Review: https://reviews.apache.org/r/50635/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d566536f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d566536f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d566536f
Branch: refs/heads/master
Commit: d566536f645ec73c41caa4ac90b232205a330a56
Parents: f291d50
Author: Joris Van Remoortere <jo...@gmail.com>
Authored: Tue Aug 2 11:33:08 2016 -0700
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Tue Aug 2 11:33:08 2016 -0700
----------------------------------------------------------------------
src/slave/status_update_manager.cpp | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d566536f/src/slave/status_update_manager.cpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.cpp b/src/slave/status_update_manager.cpp
index 9d16e0d..9c67b85 100644
--- a/src/slave/status_update_manager.cpp
+++ b/src/slave/status_update_manager.cpp
@@ -676,9 +676,13 @@ StatusUpdateStream::StatusUpdateStream(
}
// Open the updates file.
+ // NOTE: We don't use `O_SYNC` here because we only read this file
+ // if the host did not crash. `os::write` success implies the kernel
+ // will have flushed our data to the page cache. This is sufficient
+ // for the recovery scenarios we use this data for.
Try<int> result = os::open(
path.get(),
- O_CREAT | O_WRONLY | O_APPEND | O_SYNC | O_CLOEXEC,
+ O_CREAT | O_WRONLY | O_APPEND | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (result.isError()) {
[2/3] mesos git commit: Libprocess: Removed old http_parser code.
Posted by jo...@apache.org.
Libprocess: Removed old http_parser code.
We remove the code that supported the `HTTP_PARSER_VERSION_MAJOR` < 2
path.
Review: https://reviews.apache.org/r/50683
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f291d502
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f291d502
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f291d502
Branch: refs/heads/master
Commit: f291d5023e9f2e471c11d4f20590901d9bfc1de4
Parents: 2776a09
Author: Joris Van Remoortere <jo...@gmail.com>
Authored: Mon Aug 1 17:14:37 2016 -0700
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Tue Aug 2 11:31:07 2016 -0700
----------------------------------------------------------------------
3rdparty/libprocess/src/decoder.hpp | 99 ++------------------------------
1 file changed, 5 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f291d502/3rdparty/libprocess/src/decoder.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/decoder.hpp b/3rdparty/libprocess/src/decoder.hpp
index 47cfc0f..ccf12ac 100644
--- a/3rdparty/libprocess/src/decoder.hpp
+++ b/3rdparty/libprocess/src/decoder.hpp
@@ -30,6 +30,11 @@
#include <stout/try.hpp>
+#if !(HTTP_PARSER_VERSION_MAJOR >= 2)
+#error HTTP Parser version >= 2 required.
+#endif
+
+
namespace process {
// TODO(benh): Make DataDecoder abstract and make RequestDecoder a
@@ -41,13 +46,6 @@ public:
: s(_s), failure(false), request(nullptr)
{
settings.on_message_begin = &DataDecoder::on_message_begin;
-
-#if !(HTTP_PARSER_VERSION_MAJOR >= 2)
- settings.on_path = &DataDecoder::on_path;
- settings.on_fragment = &DataDecoder::on_fragment;
- settings.on_query_string = &DataDecoder::on_query_string;
-#endif
-
settings.on_url = &DataDecoder::on_url;
settings.on_header_field = &DataDecoder::on_header_field;
settings.on_header_value = &DataDecoder::on_header_value;
@@ -120,50 +118,16 @@ private:
return 0;
}
-#if !(HTTP_PARSER_VERSION_MAJOR >= 2)
- static int on_path(http_parser* p, const char* data, size_t length)
- {
- DataDecoder* decoder = (DataDecoder*) p->data;
- CHECK_NOTNULL(decoder->request);
- decoder->request->url.path.append(data, length);
- return 0;
- }
-
- static int on_query_string(http_parser* p, const char* data, size_t length)
- {
- DataDecoder* decoder = (DataDecoder*) p->data;
- CHECK_NOTNULL(decoder->request);
- decoder->query.append(data, length);
- return 0;
- }
-
- static int on_fragment(http_parser* p, const char* data, size_t length)
- {
- DataDecoder* decoder = (DataDecoder*) p->data;
- CHECK_NOTNULL(decoder->request);
-
- if (decoder->request->url.fragment.isNone()) {
- decoder->request->url.fragment = "";
- }
-
- decoder->request->url.fragment->append(data, length);
- return 0;
- }
-#endif // !(HTTP_PARSER_VERSION_MAJOR >= 2)
-
static int on_url(http_parser* p, const char* data, size_t length)
{
DataDecoder* decoder = (DataDecoder*) p->data;
CHECK_NOTNULL(decoder->request);
-#if (HTTP_PARSER_VERSION_MAJOR >= 2)
- // Reworked parsing for version >= 2.0.
// The current http_parser library (version 2.6.2 and below)
// does not support incremental parsing of URLs. To compensate
// we incrementally collect the data and parse it in
// `on_message_complete`.
decoder->url.append(data, length);
-#endif
return 0;
}
@@ -227,7 +191,6 @@ private:
CHECK_NOTNULL(decoder->request);
-#if (HTTP_PARSER_VERSION_MAJOR >= 2)
// Parse the URL. This data was incrementally built up during calls
// to `on_url`.
http_parser_url url;
@@ -256,7 +219,6 @@ private:
decoder->url.data() + url.field_data[UF_QUERY].off,
url.field_data[UF_QUERY].len);
}
-#endif
// Parse the query key/values.
Try<hashmap<std::string, std::string>> decoded =
@@ -317,13 +279,6 @@ public:
: failure(false), header(HEADER_FIELD), response(nullptr)
{
settings.on_message_begin = &ResponseDecoder::on_message_begin;
-
-#if !(HTTP_PARSER_VERSION_MAJOR >=2)
- settings.on_path = &ResponseDecoder::on_path;
- settings.on_fragment = &ResponseDecoder::on_fragment;
- settings.on_query_string = &ResponseDecoder::on_query_string;
-#endif
-
settings.on_url = &ResponseDecoder::on_url;
settings.on_header_field = &ResponseDecoder::on_header_field;
settings.on_header_value = &ResponseDecoder::on_header_value;
@@ -395,23 +350,6 @@ private:
return 0;
}
-#if !(HTTP_PARSER_VERSION_MAJOR >= 2)
- static int on_path(http_parser* p, const char* data, size_t length)
- {
- return 0;
- }
-
- static int on_query_string(http_parser* p, const char* data, size_t length)
- {
- return 0;
- }
-
- static int on_fragment(http_parser* p, const char* data, size_t length)
- {
- return 0;
- }
-#endif // !(HTTP_PARSER_VERSION_MAJOR >= 2)
-
static int on_url(http_parser* p, const char* data, size_t length)
{
return 0;
@@ -541,16 +479,6 @@ public:
{
settings.on_message_begin =
&StreamingResponseDecoder::on_message_begin;
-
-#if !(HTTP_PARSER_VERSION_MAJOR >=2)
- settings.on_path =
- &StreamingResponseDecoder::on_path;
- settings.on_fragment =
- &StreamingResponseDecoder::on_fragment;
- settings.on_query_string =
- &StreamingResponseDecoder::on_query_string;
-#endif
-
settings.on_url =
&StreamingResponseDecoder::on_url;
settings.on_header_field =
@@ -643,23 +571,6 @@ private:
return 0;
}
-#if !(HTTP_PARSER_VERSION_MAJOR >= 2)
- static int on_path(http_parser* p, const char* data, size_t length)
- {
- return 0;
- }
-
- static int on_query_string(http_parser* p, const char* data, size_t length)
- {
- return 0;
- }
-
- static int on_fragment(http_parser* p, const char* data, size_t length)
- {
- return 0;
- }
-#endif // !(HTTP_PARSER_VERSION_MAJOR >= 2)
-
static int on_status(http_parser* p, const char* data, size_t length)
{
return 0;