You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/07/07 15:08:38 UTC
[incubator-doris] branch master updated: [webserver] Make BE
webserver handle static files (#4021)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new ab8851f [webserver] Make BE webserver handle static files (#4021)
ab8851f is described below
commit ab8851f7aa7e3f496386fa659a992002f5b8470f
Author: Yingchun Lai <40...@qq.com>
AuthorDate: Tue Jul 7 23:08:29 2020 +0800
[webserver] Make BE webserver handle static files (#4021)
Make BE webserver handle static files, e.g. css, js, ico, then we can make
BE website more pretty.
---
be/src/common/config.h | 1 -
be/src/http/default_path_handlers.cpp | 8 +--
be/src/http/download_action.cpp | 81 +----------------------------
be/src/http/download_action.h | 17 +-----
be/src/http/ev_http_server.cpp | 12 +++++
be/src/http/ev_http_server.h | 4 ++
be/src/http/utils.cpp | 97 ++++++++++++++++++++++++++++++++++-
be/src/http/utils.h | 6 +++
be/src/http/web_page_handler.cpp | 87 +++++++++++++++++--------------
be/src/http/web_page_handler.h | 50 ++++++++++++------
10 files changed, 207 insertions(+), 156 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index ddb4ede..92bd75f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -535,7 +535,6 @@ namespace config {
// Whether to continue to start be when load tablet from header failed.
CONF_Bool(ignore_load_tablet_failure, "false");
-
} // namespace config
} // namespace doris
diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
index 77c8958..387de4b 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -104,10 +104,10 @@ void mem_usage_handler(MemTracker* mem_tracker, const WebPageHandler::ArgumentMa
}
void add_default_path_handlers(WebPageHandler* web_page_handler, MemTracker* process_mem_tracker) {
- web_page_handler->register_page("/logs", logs_handler);
- web_page_handler->register_page("/varz", config_handler);
- web_page_handler->register_page(
- "/memz", boost::bind<void>(&mem_usage_handler, process_mem_tracker, _1, _2));
+ web_page_handler->register_page("/logs", "Logs", logs_handler, true /* is_on_nav_bar */);
+ web_page_handler->register_page("/varz", "Configs", config_handler, true /* is_on_nav_bar */);
+ web_page_handler->register_page("/memz", "Memory",
+ boost::bind<void>(&mem_usage_handler, process_mem_tracker, _1, _2), true /* is_on_nav_bar */);
}
} // namespace doris
diff --git a/be/src/http/download_action.cpp b/be/src/http/download_action.cpp
index 47a299d..88d5ad3 100644
--- a/be/src/http/download_action.cpp
+++ b/be/src/http/download_action.cpp
@@ -18,8 +18,6 @@
#include "http/download_action.h"
#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
#include <unistd.h>
#include <string>
@@ -32,6 +30,7 @@
#include "http/http_request.h"
#include "http/http_response.h"
#include "http/http_status.h"
+#include "http/utils.h"
#include "runtime/exec_env.h"
#include "util/file_utils.h"
#include "util/filesystem_util.h"
@@ -134,84 +133,6 @@ void DownloadAction::handle(HttpRequest *req) {
LOG(INFO) << "deal with download requesst finished! ";
}
-void DownloadAction::do_dir_response(
- const std::string& dir_path, HttpRequest *req) {
- std::vector<std::string> files;
- Status status = FileUtils::list_files(Env::Default(), dir_path, &files);
- if (!status.ok()) {
- LOG(WARNING) << "Failed to scan dir. dir=" << dir_path;
- HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR);
- }
-
- const std::string FILE_DELIMETER_IN_DIR_RESPONSE = "\n";
-
- std::stringstream result;
- for (const std::string& file_name : files) {
- result << file_name << FILE_DELIMETER_IN_DIR_RESPONSE;
- }
-
- std::string result_str = result.str();
- HttpChannel::send_reply(req, result_str);
- return;
-}
-
-void DownloadAction::do_file_response(const std::string& file_path, HttpRequest *req) {
- // read file content and send response
- int fd = open(file_path.c_str(), O_RDONLY);
- if (fd < 0) {
- LOG(WARNING) << "Failed to open file: " << file_path;
- HttpChannel::send_error(req, HttpStatus::NOT_FOUND);
- return;
- }
- struct stat st;
- auto res = fstat(fd, &st);
- if (res < 0) {
- close(fd);
- LOG(WARNING) << "Failed to open file: " << file_path;
- HttpChannel::send_error(req, HttpStatus::NOT_FOUND);
- return;
- }
-
- int64_t file_size = st.st_size;
-
- // TODO(lingbin): process "IF_MODIFIED_SINCE" header
- // TODO(lingbin): process "RANGE" header
- const std::string& range_header = req->header(HttpHeaders::RANGE);
- if (!range_header.empty()) {
- // analyse range header
- }
-
- req->add_output_header(HttpHeaders::CONTENT_TYPE, get_content_type(file_path).c_str());
-
- if (req->method() == HttpMethod::HEAD) {
- close(fd);
- req->add_output_header(HttpHeaders::CONTENT_LENGTH, std::to_string(file_size).c_str());
- HttpChannel::send_reply(req);
- return;
- }
-
- HttpChannel::send_file(req, fd, 0, file_size);
-}
-
-// Do a simple decision, only deal a few type
-std::string DownloadAction::get_content_type(const std::string& file_name) {
- std::string file_ext = path_util::file_extension(file_name);
- LOG(INFO) << "file_name: " << file_name << "; file extension: [" << file_ext << "]";
- if (file_ext == std::string(".html")
- || file_ext == std::string(".htm")) {
- return std::string("text/html; charset=utf-8");
- } else if (file_ext == std::string(".js")) {
- return std::string("application/javascript; charset=utf-8");
- } else if (file_ext == std::string(".css")) {
- return std::string("text/css; charset=utf-8");
- } else if (file_ext == std::string(".txt")) {
- return std::string("text/plain; charset=utf-8");
- } else {
- return "text/plain; charset=utf-8";
- }
- return "";
-}
-
Status DownloadAction::check_token(HttpRequest *req) {
const std::string& token_str = req->param(TOKEN_PARAMETER);
if (token_str.empty()) {
diff --git a/be/src/http/download_action.h b/be/src/http/download_action.h
index a82f195..e04b94a 100644
--- a/be/src/http/download_action.h
+++ b/be/src/http/download_action.h
@@ -54,28 +54,13 @@ private:
Status check_log_path_is_allowed(const std::string& file_path);
void handle_normal(HttpRequest *req, const std::string& file_param);
- void handle_error_log(
- HttpRequest *req,
- const std::string& file_param);
-
- void do_file_response(const std::string& dir_path, HttpRequest *req);
- void do_dir_response(const std::string& dir_path, HttpRequest *req);
-
- Status get_file_content(
- FILE* fp, char* buffer, int32_t buffer_size,
- int32_t* readed_size, bool* eos);
-
- int64_t get_file_size(FILE* fp);
-
- std::string get_content_type(const std::string& file_name);
+ void handle_error_log(HttpRequest *req, const std::string& file_param);
ExecEnv* _exec_env;
DOWNLOAD_TYPE _download_type;
std::vector<std::string> _allow_paths;
std::string _error_log_root_dir;
-
-
}; // end class DownloadAction
} // end namespace doris
diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp
index 48013d4..ee8684d 100644
--- a/be/src/http/ev_http_server.cpp
+++ b/be/src/http/ev_http_server.cpp
@@ -206,6 +206,14 @@ bool EvHttpServer::register_handler(
return result;
}
+void EvHttpServer::register_static_file_handler(HttpHandler* handler) {
+ DCHECK(handler != nullptr);
+ DCHECK(_static_file_handler == nullptr);
+ pthread_rwlock_wrlock(&_rw_lock);
+ _static_file_handler = handler;
+ pthread_rwlock_unlock(&_rw_lock);
+}
+
int EvHttpServer::on_header(struct evhttp_request* ev_req) {
std::unique_ptr<HttpRequest> request(new HttpRequest(ev_req));
auto res = request->init_from_evhttp();
@@ -247,6 +255,10 @@ HttpHandler* EvHttpServer::_find_handler(HttpRequest* req) {
switch (req->method()) {
case GET:
_get_handlers.retrieve(path, &handler, req->params());
+ // Static file handler is a fallback handler
+ if (handler == nullptr) {
+ handler = _static_file_handler;
+ }
break;
case PUT:
_put_handlers.retrieve(path, &handler, req->params());
diff --git a/be/src/http/ev_http_server.h b/be/src/http/ev_http_server.h
index 466104e..6ef632e 100644
--- a/be/src/http/ev_http_server.h
+++ b/be/src/http/ev_http_server.h
@@ -39,6 +39,9 @@ public:
// register handler for an a path-method pair
bool register_handler(
const HttpMethod& method, const std::string& path, HttpHandler* handler);
+
+ void register_static_file_handler(HttpHandler* handler);
+
Status start();
void stop();
void join();
@@ -69,6 +72,7 @@ private:
pthread_rwlock_t _rw_lock;
PathTrie<HttpHandler*> _get_handlers;
+ HttpHandler* _static_file_handler = nullptr;
PathTrie<HttpHandler*> _put_handlers;
PathTrie<HttpHandler*> _post_handlers;
PathTrie<HttpHandler*> _delete_handlers;
diff --git a/be/src/http/utils.cpp b/be/src/http/utils.cpp
index 7953a7c..bc8bfbf 100644
--- a/be/src/http/utils.cpp
+++ b/be/src/http/utils.cpp
@@ -15,13 +15,21 @@
// specific language governing permissions and limitations
// under the License.
-#include <http/utils.h>
+#include "http/utils.h"
+
+#include <fcntl.h>
+#include <sys/stat.h>
#include "common/logging.h"
+#include "common/status.h"
#include "common/utils.h"
+#include "env/env.h"
+#include "util/file_utils.h"
#include "http/http_common.h"
+#include "http/http_channel.h"
#include "http/http_headers.h"
#include "http/http_request.h"
+#include "util/path_util.h"
#include "util/url_coding.h"
namespace doris {
@@ -78,4 +86,91 @@ bool parse_basic_auth(const HttpRequest& req, AuthInfo* auth) {
return true;
}
+// Do a simple decision, only deal a few type
+std::string get_content_type(const std::string& file_name) {
+ std::string file_ext = path_util::file_extension(file_name);
+ LOG(INFO) << "file_name: " << file_name << "; file extension: [" << file_ext << "]";
+ if (file_ext == std::string(".html")
+ || file_ext == std::string(".htm")) {
+ return std::string("text/html; charset=utf-8");
+ } else if (file_ext == std::string(".js")) {
+ return std::string("application/javascript; charset=utf-8");
+ } else if (file_ext == std::string(".css")) {
+ return std::string("text/css; charset=utf-8");
+ } else if (file_ext == std::string(".txt")) {
+ return std::string("text/plain; charset=utf-8");
+ } else if (file_ext == std::string(".png")) {
+ return std::string("image/png");
+ } else if (file_ext == std::string(".ico")) {
+ return std::string("image/x-icon");
+ } else {
+ return "text/plain; charset=utf-8";
+ }
+ return "";
+}
+
+void do_file_response(const std::string& file_path, HttpRequest *req) {
+ if (file_path.find("..") != std::string::npos) {
+ LOG(WARNING) << "Not allowed to read relative path: " << file_path;
+ HttpChannel::send_error(req, HttpStatus::FORBIDDEN);
+ return;
+ }
+
+ // read file content and send response
+ int fd = open(file_path.c_str(), O_RDONLY);
+ if (fd < 0) {
+ LOG(WARNING) << "Failed to open file: " << file_path;
+ HttpChannel::send_error(req, HttpStatus::NOT_FOUND);
+ return;
+ }
+ struct stat st;
+ auto res = fstat(fd, &st);
+ if (res < 0) {
+ close(fd);
+ LOG(WARNING) << "Failed to open file: " << file_path;
+ HttpChannel::send_error(req, HttpStatus::NOT_FOUND);
+ return;
+ }
+
+ int64_t file_size = st.st_size;
+
+ // TODO(lingbin): process "IF_MODIFIED_SINCE" header
+ // TODO(lingbin): process "RANGE" header
+ const std::string& range_header = req->header(HttpHeaders::RANGE);
+ if (!range_header.empty()) {
+ // analyse range header
+ }
+
+ req->add_output_header(HttpHeaders::CONTENT_TYPE, get_content_type(file_path).c_str());
+
+ if (req->method() == HttpMethod::HEAD) {
+ close(fd);
+ req->add_output_header(HttpHeaders::CONTENT_LENGTH, std::to_string(file_size).c_str());
+ HttpChannel::send_reply(req);
+ return;
+ }
+
+ HttpChannel::send_file(req, fd, 0, file_size);
+}
+
+void do_dir_response(const std::string& dir_path, HttpRequest *req) {
+ std::vector<std::string> files;
+ Status status = FileUtils::list_files(Env::Default(), dir_path, &files);
+ if (!status.ok()) {
+ LOG(WARNING) << "Failed to scan dir. dir=" << dir_path;
+ HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR);
+ }
+
+ const std::string FILE_DELIMETER_IN_DIR_RESPONSE = "\n";
+
+ std::stringstream result;
+ for (const std::string& file_name : files) {
+ result << file_name << FILE_DELIMETER_IN_DIR_RESPONSE;
+ }
+
+ std::string result_str = result.str();
+ HttpChannel::send_reply(req, result_str);
+ return;
+}
+
}
diff --git a/be/src/http/utils.h b/be/src/http/utils.h
index 8e82d7b..0a1a50b 100644
--- a/be/src/http/utils.h
+++ b/be/src/http/utils.h
@@ -21,6 +21,7 @@
#include "common/utils.h"
#include "http/http_common.h"
+#include "http/http_request.h"
namespace doris {
@@ -34,4 +35,9 @@ bool parse_basic_auth(const HttpRequest& req, std::string* user, std::string* pa
bool parse_basic_auth(const HttpRequest& req, AuthInfo* auth);
+void do_file_response(const std::string& dir_path, HttpRequest *req);
+
+void do_dir_response(const std::string& dir_path, HttpRequest *req);
+
+std::string get_content_type(const std::string& file_name);
}
diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp
index 2140b27..cd31853 100644
--- a/be/src/http/web_page_handler.cpp
+++ b/be/src/http/web_page_handler.cpp
@@ -20,80 +20,91 @@
#include <boost/bind.hpp>
#include <boost/mem_fn.hpp>
+#include "common/config.h"
+#include "env/env.h"
+#include "gutil/stl_util.h"
+#include "gutil/strings/substitute.h"
#include "http/ev_http_server.h"
#include "http/http_channel.h"
#include "http/http_headers.h"
#include "http/http_request.h"
#include "http/http_response.h"
#include "http/http_status.h"
+#include "http/utils.h"
+#include "olap/file_helper.h"
#include "util/cpu_info.h"
#include "util/debug_util.h"
#include "util/disk_info.h"
#include "util/mem_info.h"
+using strings::Substitute;
+
namespace doris {
static std::string s_html_content_type = "text/html";
WebPageHandler::WebPageHandler(EvHttpServer* server) : _http_server(server) {
- PageHandlerCallback default_callback =
+ // Make WebPageHandler to be static file handler, static files, e.g. css, png, will be handled by WebPageHandler.
+ _http_server->register_static_file_handler(this);
+
+ PageHandlerCallback root_callback =
boost::bind<void>(boost::mem_fn(&WebPageHandler::root_handler), this, _1, _2);
- register_page("/", default_callback);
+ register_page("/", "Home", root_callback, false /* is_on_nav_bar */);
+}
+
+WebPageHandler::~WebPageHandler() {
+ STLDeleteValues(&_page_map);
}
-void WebPageHandler::register_page(const std::string& path, const PageHandlerCallback& callback) {
- // Put this handler to to s_handler_by_name
- // because handler does't often new
- // So we insert it to this set when everytime
+void WebPageHandler::register_page(const std::string& path, const string& alias,
+ const PageHandlerCallback& callback, bool is_on_nav_bar) {
boost::mutex::scoped_lock lock(_map_lock);
- auto map_iter = _page_map.find(path);
- if (map_iter == _page_map.end()) {
- // first time, register this to web server
- _http_server->register_handler(HttpMethod::GET, path, this);
- }
- _page_map[path].add_callback(callback);
+ CHECK(_page_map.find(path) == _page_map.end());
+ // first time, register this to web server
+ _http_server->register_handler(HttpMethod::GET, path, this);
+ _page_map[path] = new PathHandler(true /* is_styled */, is_on_nav_bar, alias, callback);
}
void WebPageHandler::handle(HttpRequest* req) {
- // Should we render with css styles?
- bool use_style = true;
- auto& params = *req->params();
- if (params.find("raw") != params.end()) {
- use_style = false;
- }
-
- std::stringstream output;
-
- // Append header
- if (use_style) {
- bootstrap_page_header(&output);
- }
-
- // Append content
- // push_content(&output);
LOG(INFO) << req->debug_string();
+
+ PathHandler* handler = nullptr;
{
boost::mutex::scoped_lock lock(_map_lock);
auto iter = _page_map.find(req->raw_path());
if (iter != _page_map.end()) {
- for (auto& callback : iter->second.callbacks()) {
- callback(*req->params(), &output);
- }
+ handler = iter->second;
}
}
+ if (handler == nullptr) {
+ // Try to handle static file request
+ do_file_response(std::string(getenv("DORIS_HOME")) + "/www/" + req->raw_path(), req);
+ // Has replied in do_file_response, so we return here.
+ return;
+ }
+
+ const auto& params = *req->params();
+
+ // Should we render with css styles?
+ bool use_style = (params.find("raw") == params.end());
+
+ std::stringstream content;
+ // Append header
+ if (use_style) {
+ bootstrap_page_header(&content);
+ }
+
+ // Append content
+ handler->callback()(params, &content);
+
// Append footer
if (use_style) {
- bootstrap_page_footer(&output);
+ bootstrap_page_footer(&content);
}
- std::string str = output.str();
req->add_output_header(HttpHeaders::CONTENT_TYPE, s_html_content_type.c_str());
- HttpChannel::send_reply(req, HttpStatus::OK, str);
-#if 0
- HttpResponse response(HttpStatus::OK, s_html_content_type, &str);
- channel->send_response(response);
-#endif
+ HttpChannel::send_reply(req, HttpStatus::OK, content.str());
}
static const std::string PAGE_HEADER =
diff --git a/be/src/http/web_page_handler.h b/be/src/http/web_page_handler.h
index 2c53947..62d2f5c 100644
--- a/be/src/http/web_page_handler.h
+++ b/be/src/http/web_page_handler.h
@@ -37,34 +37,52 @@ class EvHttpServer;
class WebPageHandler : public HttpHandler {
public:
typedef std::map<std::string, std::string> ArgumentMap;
- typedef boost::function<void (const ArgumentMap& args, std::stringstream* output)>
+ typedef boost::function<void (const ArgumentMap& args, std::stringstream* output)>
PageHandlerCallback;
- WebPageHandler(EvHttpServer* http_server);
- virtual ~WebPageHandler() {
- }
+ WebPageHandler(EvHttpServer* http_server);
+ virtual ~WebPageHandler();
void handle(HttpRequest *req) override;
- // Just use old code
- void register_page(const std::string& path, const PageHandlerCallback& callback);
+ // Register a route 'path'.
+ // If 'is_on_nav_bar' is true, a link to the page will be placed on the navbar
+ // in the header of styled pages. The link text is given by 'alias'.
+ void register_page(const std::string& path, const std::string& alias,
+ const PageHandlerCallback& callback, bool is_on_nav_bar);
private:
void bootstrap_page_header(std::stringstream* output);
void bootstrap_page_footer(std::stringstream* output);
void root_handler(const ArgumentMap& args, std::stringstream* output);
- // all
- class PageHandlers {
+ // Container class for a list of path handler callbacks for a single URL.
+ class PathHandler {
public:
- void add_callback(const PageHandlerCallback& callback) {
- _callbacks.push_back(callback);
- }
- const std::vector<PageHandlerCallback>& callbacks() const {
- return _callbacks;
- }
+ PathHandler(bool is_styled, bool is_on_nav_bar, std::string alias,
+ PageHandlerCallback callback)
+ : is_styled_(is_styled),
+ is_on_nav_bar_(is_on_nav_bar),
+ alias_(std::move(alias)),
+ callback_(std::move(callback)) {}
+
+ bool is_styled() const { return is_styled_; }
+ bool is_on_nav_bar() const { return is_on_nav_bar_; }
+ const std::string& alias() const { return alias_; }
+ const PageHandlerCallback& callback() const { return callback_; }
+
private:
- std::vector<PageHandlerCallback> _callbacks;
+ // If true, the page appears is rendered styled.
+ bool is_styled_;
+
+ // If true, the page appears in the navigation bar.
+ bool is_on_nav_bar_;
+
+ // Alias used when displaying this link on the nav bar.
+ std::string alias_;
+
+ // Callback to render output for this page.
+ PageHandlerCallback callback_;
};
EvHttpServer* _http_server;
@@ -73,7 +91,7 @@ private:
// Map of path to a PathHandler containing a list of handlers for that
// path. More than one handler may register itself with a path so that many
// components may contribute to a single page.
- typedef std::map<std::string, PageHandlers> PageHandlersMap;
+ typedef std::map<std::string, PathHandler*> PageHandlersMap;
PageHandlersMap _page_map;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org