You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/06/13 06:37:15 UTC
[impala] 01/07: Update mustache to commit
b290952d8eb93d085214d8c8c9eab8559df9f606
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 25c8b6226ba90e0fee7bdd61ae5313570ce28754
Author: Todd Lipcon <to...@apache.org>
AuthorDate: Fri Jun 7 14:30:51 2019 -0700
Update mustache to commit b290952d8eb93d085214d8c8c9eab8559df9f606
This fixes some issues with regard to template variable scoping that
will be useful for Knox integration.
Change-Id: If26f3aaa2a3279a1f6a300c4f4cee7ec899e22ed
Reviewed-on: http://gerrit.cloudera.org:8080/13563
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/thirdparty/mustache/README | 2 +-
be/src/thirdparty/mustache/mustache.cc | 280 +++++++++++++++++++++------------
be/src/thirdparty/mustache/mustache.h | 2 +-
be/src/util/webserver.cc | 4 +-
4 files changed, 185 insertions(+), 103 deletions(-)
diff --git a/be/src/thirdparty/mustache/README b/be/src/thirdparty/mustache/README
index 9834079..ba94e4d 100644
--- a/be/src/thirdparty/mustache/README
+++ b/be/src/thirdparty/mustache/README
@@ -1,3 +1,3 @@
This Mustache template library is taken from the Apache licensed version at https://github.com/henryr/cpp-mustache
-The current commit is: 80cbe5
+The current commit is: b290952d8eb93d085214d8c8c9eab8559df9f606
diff --git a/be/src/thirdparty/mustache/mustache.cc b/be/src/thirdparty/mustache/mustache.cc
index fc6c4f6..e976677 100644
--- a/be/src/thirdparty/mustache/mustache.cc
+++ b/be/src/thirdparty/mustache/mustache.cc
@@ -13,14 +13,15 @@
#include "mustache.h"
#include "rapidjson/stringbuffer.h"
+#include <rapidjson/prettywriter.h>
#include "rapidjson/writer.h"
#include <iostream>
#include <fstream>
#include <vector>
+#include <stack>
#include <boost/algorithm/string.hpp>
-#include <boost/foreach.hpp>
using namespace rapidjson;
using namespace std;
@@ -30,7 +31,6 @@ namespace mustache {
// TODO:
// # Handle malformed templates better
-// # Support array_tag.length?
// # Better support for reading templates from files
enum TagOperator {
@@ -42,9 +42,24 @@ enum TagOperator {
PARTIAL,
COMMENT,
LENGTH,
+ EQUALITY,
+ INEQUALITY,
+ LITERAL,
NONE
};
+struct OpCtx {
+ TagOperator op;
+ string tag_name;
+ string tag_arg;
+ bool escaped = false;
+};
+
+struct ContextStack {
+ const Value* value;
+ const ContextStack* parent;
+};
+
TagOperator GetOperator(const string& tag) {
if (tag.size() == 0) return SUBSTITUTION;
switch (tag[0]) {
@@ -53,18 +68,24 @@ TagOperator GetOperator(const string& tag) {
case '?': return PREDICATE_SECTION_START;
case '/': return SECTION_END;
case '>': return PARTIAL;
- case '!': return COMMENT;
+ case '!':
+ if (tag.size() == 1 || tag[1] != '=') return COMMENT;
+ return INEQUALITY;
case '%': return LENGTH;
+ case '~': return LITERAL;
+ case '=': return EQUALITY;
default: return SUBSTITUTION;
}
}
int EvaluateTag(const string& document, const string& document_root, int idx,
- const Value* context, TagOperator tag, const string& tag_name, bool is_triple,
- stringstream* out);
+ const ContextStack* context, const OpCtx& op_ctx, stringstream* out);
+
+static bool RenderTemplate(const string& document, const string& document_root,
+ const ContextStack* stack, stringstream* out);
void EscapeHtml(const string& in, stringstream *out) {
- BOOST_FOREACH(const char& c, in) {
+ for (const char& c: in) {
switch (c) {
case '&': (*out) << "&";
break;
@@ -82,6 +103,13 @@ void EscapeHtml(const string& in, stringstream *out) {
}
}
+void Dump(const rapidjson::Value& v) {
+ StringBuffer buffer;
+ Writer<StringBuffer> writer(buffer);
+ v.Accept(writer);
+ std::cout << buffer.GetString() << std::endl;
+}
+
// Breaks a dotted path into individual components. One wrinkle, which stops this from
// being a simple split() is that we allow path components to be quoted, e.g.: "foo".bar,
// and any '.' characters inside those quoted sections aren't considered to be
@@ -122,38 +150,45 @@ void FindJsonPathComponents(const string& path, vector<string>* components) {
}
// Looks up the json entity at 'path' in 'parent_context', and places it in 'resolved'. If
-// the entity does not exist (i.e. the path is invalid), 'resolved' will be set to NULL.
-void ResolveJsonContext(const string& path, const Value& parent_context,
+// the entity does not exist (i.e. the path is invalid), 'resolved' will be set to nullptr.
+void ResolveJsonContext(const string& path, const ContextStack* stack,
const Value** resolved) {
if (path == ".") {
- *resolved = &parent_context;
+ *resolved = stack->value;
return;
}
vector<string> components;
FindJsonPathComponents(path, &components);
- const Value* cur = &parent_context;
- BOOST_FOREACH(const string& c, components) {
- if (cur->IsObject() && cur->HasMember(c.c_str())) {
- cur = &(*cur)[c.c_str()];
- } else {
- *resolved = NULL;
+
+ // At each enclosing level of context, try to resolve the path.
+ for ( ; stack != nullptr; stack = stack->parent) {
+ const Value* cur = stack->value;
+ bool match = true;
+ for(const string& c: components) {
+ if (cur->IsObject() && cur->HasMember(c.c_str())) {
+ cur = &(*cur)[c.c_str()];
+ } else {
+ match = false;
+ break;
+ }
+ }
+ if (match) {
+ *resolved = cur;
return;
}
}
-
- *resolved = cur;
+ *resolved = nullptr;
}
-int FindNextTag(const string& document, int idx, TagOperator* tag_op, string* tag_name,
- bool* is_triple, stringstream* out) {
- *tag_op = NONE;
+int FindNextTag(const string& document, int idx, OpCtx* op, stringstream* out) {
+ op->op = NONE;
while (idx < document.size()) {
if (document[idx] == '{' && idx < (document.size() - 3) && document[idx + 1] == '{') {
if (document[idx + 2] == '{') {
idx += 3;
- *is_triple = true;
+ op->escaped = true;
} else {
- *is_triple = false;
+ op->escaped = false;
idx += 2; // Now at start of template expression
}
stringstream expr;
@@ -162,10 +197,10 @@ int FindNextTag(const string& document, int idx, TagOperator* tag_op, string* ta
expr << document[idx];
++idx;
} else {
- if (!*is_triple && idx < document.size() - 1 && document[idx + 1] == '}') {
+ if (!op->escaped && idx < document.size() - 1 && document[idx + 1] == '}') {
++idx;
break;
- } else if (*is_triple && idx < document.size() - 2 && document[idx + 1] == '}'
+ } else if (op->escaped && idx < document.size() - 2 && document[idx + 1] == '}'
&& document[idx + 2] == '}') {
idx += 2;
break;
@@ -179,16 +214,27 @@ int FindNextTag(const string& document, int idx, TagOperator* tag_op, string* ta
trim(key);
if (key != ".") trim_if(key, is_any_of("."));
if (key.size() == 0) continue;
- *tag_op = GetOperator(key);
- if (*tag_op != SUBSTITUTION) {
- key = key.substr(1);
+ op->op = GetOperator(key);
+ if (op->op != SUBSTITUTION) {
+ int len = op->op == INEQUALITY ? 2 : 1;
+ key = key.substr(len);
trim(key);
}
if (key.size() == 0) continue;
- *tag_name = key;
+
+ if (op->op == EQUALITY || op->op == INEQUALITY) {
+ // Find an argument
+ vector<string> components;
+ split(components, key, is_any_of(" "));
+ key = components[0];
+ components.erase(components.begin());
+ op->tag_arg = join(components, " ");
+ }
+
+ op->tag_name = key;
return ++idx;
} else {
- if (out != NULL) (*out) << document[idx];
+ if (out != nullptr) (*out) << document[idx];
}
++idx;
}
@@ -203,57 +249,69 @@ int FindNextTag(const string& document, int idx, TagOperator* tag_op, string* ta
// If 'is_negation' is true, the behaviour is the opposite of the above: false values
// cause the section to be normally evaluated etc.
int EvaluateSection(const string& document, const string& document_root, int idx,
- const Value* parent_context, TagOperator op, const string& tag_name,
- stringstream* out) {
- // Precondition: idx is the immedate next character after an opening {{ #tag_name }}
+ const ContextStack* context_stack, const OpCtx& op_ctx, stringstream* out) {
+ // Precondition: idx is the immediate next character after an opening {{ #tag_name }}
const Value* context;
- ResolveJsonContext(tag_name, *parent_context, &context);
+ ResolveJsonContext(op_ctx.tag_name, context_stack, &context);
// If we a) cannot resolve the context from the tag name or b) the context evaluates to
// false, we should skip the contents of the template until a closing {{/tag_name}}.
- bool skip_contents = (context == NULL || context->IsFalse());
-
- // If the tag is a negative block (i.e. {{^tag_name}}), do the opposite: if the context
- // exists and is true, skip the contents, else echo them.
- if (op == NEGATED_SECTION_START) {
- context = parent_context;
- skip_contents = !skip_contents;
- } else if (op == PREDICATE_SECTION_START) {
- context = parent_context;
+ bool skip_contents = false;
+
+ if (op_ctx.op == NEGATED_SECTION_START || op_ctx.op == PREDICATE_SECTION_START ||
+ op_ctx.op == SECTION_START) {
+ skip_contents = (context == nullptr || context->IsFalse());
+
+ // If the tag is a negative block (i.e. {{^tag_name}}), do the opposite: if the
+ // context exists and is true, skip the contents, else echo them.
+ if (op_ctx.op == NEGATED_SECTION_START) {
+ context = context_stack->value;
+ skip_contents = !skip_contents;
+ } else if (op_ctx.op == PREDICATE_SECTION_START) {
+ context = context_stack->value;
+ }
+ } else if (op_ctx.op == INEQUALITY || op_ctx.op == EQUALITY) {
+ skip_contents = (context == nullptr || !context->IsString() ||
+ strcasecmp(context->GetString(), op_ctx.tag_arg.c_str()) != 0);
+ if (op_ctx.op == INEQUALITY) skip_contents = !skip_contents;
+ context = context_stack->value;
}
vector<const Value*> values;
- if (!skip_contents && context != NULL && context->IsArray()) {
+ if (!skip_contents && context != nullptr && context->IsArray()) {
for (int i = 0; i < context->Size(); ++i) {
values.push_back(&(*context)[i]);
}
} else {
- values.push_back(skip_contents ? NULL : context);
+ values.push_back(skip_contents ? nullptr : context);
}
if (values.size() == 0) {
skip_contents = true;
- values.push_back(NULL);
+ values.push_back(nullptr);
}
int start_idx = idx;
- BOOST_FOREACH(const Value* v, values) {
+ for(const Value* v: values) {
idx = start_idx;
+ stack<OpCtx> section_starts;
+ section_starts.push(op_ctx);
while (idx < document.size()) {
- TagOperator tag_op;
- string next_tag_name;
- bool is_triple;
- idx = FindNextTag(document, idx, &tag_op, &next_tag_name, &is_triple,
- skip_contents ? NULL : out);
-
- if (idx > document.size()) return idx;
- if (tag_op == SECTION_END && next_tag_name == tag_name) {
- break;
+ OpCtx next_ctx;
+ idx = FindNextTag(document, idx, &next_ctx, skip_contents ? nullptr : out);
+ if (skip_contents && (next_ctx.op == SECTION_START ||
+ next_ctx.op == PREDICATE_SECTION_START ||
+ next_ctx.op == NEGATED_SECTION_START)) {
+ section_starts.push(next_ctx);
+ } else if (next_ctx.op == SECTION_END) {
+ if (next_ctx.tag_name != section_starts.top().tag_name) return -1;
+ section_starts.pop();
}
+ if (section_starts.empty()) break;
// Don't need to evaluate any templates if we're skipping the contents
if (!skip_contents) {
- idx = EvaluateTag(document, document_root, idx, v, tag_op, next_tag_name,
- is_triple, out);
+ ContextStack new_context = { v, context_stack };
+ idx = EvaluateTag(document, document_root, idx, &new_context, next_ctx, out);
}
}
}
@@ -263,53 +321,65 @@ int EvaluateSection(const string& document, const string& document_root, int idx
// Evaluates a SUBSTITUTION tag, by replacing its contents with the value of the tag's
// name in 'parent_context'.
int EvaluateSubstitution(const string& document, const int idx,
- const Value* parent_context, const string& tag_name, bool is_triple,
- stringstream* out) {
- const Value* context;
- ResolveJsonContext(tag_name, *parent_context, &context);
- if (context == NULL) return idx;
- if (context->IsString()) {
- if (!is_triple) {
- EscapeHtml(context->GetString(), out);
+ const ContextStack* context_stack, const OpCtx& op_ctx, stringstream* out) {
+ const Value* val;
+ ResolveJsonContext(op_ctx.tag_name, context_stack, &val);
+ if (val == nullptr) return idx;
+ if (val->IsString()) {
+ if (!op_ctx.escaped) {
+ EscapeHtml(val->GetString(), out);
} else {
// TODO: Triple {{{ means don't escape
- (*out) << context->GetString();
+ (*out) << val->GetString();
}
- } else if (context->IsInt64()) {
- (*out) << context->GetInt64();
- } else if (context->IsInt()) {
- (*out) << context->GetInt();
- } else if (context->IsDouble()) {
- (*out) << context->GetDouble();
- } else if (context->IsBool()) {
- (*out) << boolalpha << context->GetBool();
+ } else if (val->IsInt64()) {
+ (*out) << val->GetInt64();
+ } else if (val->IsInt()) {
+ (*out) << val->GetInt();
+ } else if (val->IsDouble()) {
+ (*out) << val->GetDouble();
+ } else if (val->IsBool()) {
+ (*out) << boolalpha << val->GetBool();
}
return idx;
}
// Evaluates a LENGTH tag by replacing its contents with the type-dependent 'size' of the
// value.
-int EvaluateLength(const string& document, const int idx, const Value* parent_context,
+int EvaluateLength(const string& document, const int idx, const ContextStack* context_stack,
const string& tag_name, stringstream* out) {
- const Value* context;
- ResolveJsonContext(tag_name, *parent_context, &context);
- if (context == NULL) return idx;
- if (context->IsArray()) {
- (*out) << context->Size();
- } else if (context->IsString()) {
- (*out) << context->GetStringLength();
+ const Value* val;
+ ResolveJsonContext(tag_name, context_stack, &val);
+ if (val == nullptr) return idx;
+ if (val->IsArray()) {
+ (*out) << val->Size();
+ } else if (val->IsString()) {
+ (*out) << val->GetStringLength();
};
return idx;
}
+int EvaluateLiteral(const string& document, const int idx, const ContextStack* context_stack,
+ const string& tag_name, stringstream* out) {
+ const Value* val;
+ ResolveJsonContext(tag_name, context_stack, &val);
+ if (val == nullptr) return idx;
+ if (!val->IsArray() && !val->IsObject()) return idx;
+ StringBuffer strbuf;
+ PrettyWriter<StringBuffer> writer(strbuf);
+ val->Accept(writer);
+ (*out) << strbuf.GetString();
+ return idx;
+}
+
// Evaluates a 'partial' template by reading it fully from disk, then rendering it
// directly into the current output with the current context.
//
// TODO: This could obviously be more efficient (and there are lots of file accesses in a
// long list context).
void EvaluatePartial(const string& tag_name, const string& document_root,
- const Value* parent_context, stringstream* out) {
+ const ContextStack* stack, stringstream* out) {
stringstream ss;
ss << document_root << tag_name;
ifstream tmpl(ss.str().c_str());
@@ -320,49 +390,59 @@ void EvaluatePartial(const string& tag_name, const string& document_root,
}
stringstream file_ss;
file_ss << tmpl.rdbuf();
- RenderTemplate(file_ss.str(), document_root, *parent_context, out);
+ RenderTemplate(file_ss.str(), document_root, stack, out);
}
// Given a tag name, and its operator, evaluate the tag in the given context and write the
// output to 'out'. The heavy-lifting is delegated to specific Evaluate*()
// methods. Returns the new cursor position within 'document', or -1 on error.
int EvaluateTag(const string& document, const string& document_root, int idx,
- const Value* context, TagOperator tag,
- const string& tag_name, bool is_triple, stringstream* out) {
+ const ContextStack* context, const OpCtx& op_ctx, stringstream* out) {
if (idx == -1) return idx;
- switch (tag) {
+ switch (op_ctx.op) {
case SECTION_START:
case PREDICATE_SECTION_START:
case NEGATED_SECTION_START:
- return EvaluateSection(document, document_root, idx, context, tag, tag_name, out);
+ case EQUALITY:
+ case INEQUALITY:
+ return EvaluateSection(document, document_root, idx, context, op_ctx, out);
case SUBSTITUTION:
- return EvaluateSubstitution(document, idx, context, tag_name, is_triple, out);
+ return EvaluateSubstitution(document, idx, context, op_ctx, out);
case COMMENT:
return idx; // Ignored
case PARTIAL:
- EvaluatePartial(tag_name, document_root, context, out);
+ EvaluatePartial(op_ctx.tag_name, document_root, context, out);
return idx;
case LENGTH:
- return EvaluateLength(document, idx, context, tag_name, out);
+ return EvaluateLength(document, idx, context, op_ctx.tag_name, out);
+ case LITERAL:
+ return EvaluateLiteral(document, idx, context, op_ctx.tag_name, out);
case NONE:
return idx; // No tag was found
+ case SECTION_END:
+ return idx;
default:
- cout << "Unknown tag: " << tag << endl;
+ cout << "Unknown tag: " << op_ctx.op << endl;
return -1;
}
}
-void RenderTemplate(const string& document, const string& document_root,
- const Value& context, stringstream* out) {
+static bool RenderTemplate(const string& document, const string& document_root,
+ const ContextStack* stack, stringstream* out) {
int idx = 0;
while (idx < document.size() && idx != -1) {
- string tag_name;
- TagOperator tag_op;
- bool is_triple = true;
- idx = FindNextTag(document, idx, &tag_op, &tag_name, &is_triple, out);
- idx = EvaluateTag(document, document_root, idx, &context, tag_op, tag_name, is_triple,
- out);
+ OpCtx op;
+ idx = FindNextTag(document, idx, &op, out);
+ idx = EvaluateTag(document, document_root, idx, stack, op, out);
}
+
+ return idx != -1;
+}
+
+bool RenderTemplate(const string& document, const string& document_root,
+ const Value& context, stringstream* out) {
+ ContextStack stack = { &context, nullptr };
+ return RenderTemplate(document, document_root, &stack, out);
}
}
diff --git a/be/src/thirdparty/mustache/mustache.h b/be/src/thirdparty/mustache/mustache.h
index e4168d3..9a2d9fb 100644
--- a/be/src/thirdparty/mustache/mustache.h
+++ b/be/src/thirdparty/mustache/mustache.h
@@ -21,7 +21,7 @@ namespace mustache {
// 'context'. Alternately finds a tag and then evaluates it. Returns when an error is
// signalled (TODO: probably doesn't work in all paths), and evaluates that tag. Output is
// accumulated in 'out'.
-void RenderTemplate(const std::string& document, const std::string& document_root,
+bool RenderTemplate(const std::string& document, const std::string& document_root,
const rapidjson::Value& context, std::stringstream* out);
}
diff --git a/be/src/util/webserver.cc b/be/src/util/webserver.cc
index 331c6d9..3ea3e65 100644
--- a/be/src/util/webserver.cc
+++ b/be/src/util/webserver.cc
@@ -541,8 +541,10 @@ void Webserver::RenderUrlWithTemplate(const WebRequest& req,
} else {
stringstream buffer;
buffer << tmpl.rdbuf();
- RenderTemplate(buffer.str(), Substitute("$0/", FLAGS_webserver_doc_root), document,
+ bool success = RenderTemplate(buffer.str(),
+ Substitute("$0/", FLAGS_webserver_doc_root), document,
output);
+ LOG_IF(WARNING, !success) << "could not render template " << full_template_path;
}
}
}