You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by GitBox <gi...@apache.org> on 2020/07/29 17:47:58 UTC
[GitHub] [incubator-tvm] ANSHUMAN87 commented on a change in pull request #6162: [Parser] Parser 2.0 part 2
ANSHUMAN87 commented on a change in pull request #6162:
URL: https://github.com/apache/incubator-tvm/pull/6162#discussion_r462106658
##########
File path: src/parser/meta_ref.h
##########
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file meta_ref.h
+ * \brief A reference into the metadata section of the Relay text format.
+ */
+
+#ifndef TVM_PARSER_META_REF_H_
+#define TVM_PARSER_META_REF_H_
+
+
+#include <tvm/ir/attrs.h>
+#include <tvm/relay/expr.h>
+
+#include <string>
+
+namespace tvm {
+namespace parser {
+
+using namespace relay;
+
+using MetaTable = Map<String, Array<ObjectRef>>;
+
+/*!
+ * \brief Options for allocating storage.
+ */
+struct MetaRefAttrs : public tvm::AttrsNode<MetaRefAttrs> {
+ tvm::String node_type_key;
+ uint64_t node_index;
+
+ TVM_DECLARE_ATTRS(MetaRefAttrs, "relay.attrs.MetaRefAttrs") {
+ TVM_ATTR_FIELD(node_type_key)
+ .describe("The type_key representing the type of the node referenced.");
+ TVM_ATTR_FIELD(node_index).describe("The index into the type specific node array.");
+ }
+};
+
+/*! \brief A reference to a "meta-expression".
+ *
+ * In the text format we allow referencing metadata which
+ * uses a compact serialization that proceeds the main
+ * program body.
+ *
+ * We can reference this table using an expression of
+ * the form `meta[Type][index]`.
+ *
+ * We must later resolve these references to actual in-memory
+ * AST nodes but this requires first parsing the full program
+ * then expanding these temporary AST nodes into their corresponding
+ * nodes.
+ *
+ * For example the nth large constant will be pretty-printed as meta[relay.Constant][n]
+ * with its compact binary serialization residing in the metadata section at the end
+ * of the program.
+ *
+ * \param type_key The type key of the object in the meta section.
+ * \param kind The index into that subfield.
Review comment:
kind -> node_index
##########
File path: src/parser/diagnostic.h
##########
@@ -138,8 +62,73 @@ struct Diagnostic {
/*! \brief The diagnostic message. */
std::string message;
+ /*! \brief A diagnostic for a single character token. */
Diagnostic(int line, int column, const std::string& message)
- : level(DiagnosticLevel::Error), span(SourceName(), line, column), message(message) {}
+ : level(DiagnosticLevel::Error), span(SourceName(), line, column, line, column + 1), message(message) {}
+
+ Diagnostic(DiagnosticLevel level, Span span, const std::string& message)
+ : level(level), span(span), message(message) {}
+};
+
+/*!
+ * \brief A wrapper around std::stringstream to build a diagnostic.
+ *
+ * \code
+ *
+ * void ReportError(const Error& err);
+ *
+ * void Test(int number) {
+ * // Use error reporter to construct an error.
+ * ReportError(ErrorBuilder() << "This is an error number=" << number);
+ * }
+ *
+ * \endcode
+ */
+struct DiagnosticBuilder {
+ public:
+ /*! \brief The level. */
+ DiagnosticLevel level;
+
+ /*! \brief The source name. */
+ SourceName source_name;
+
+ /*! \brief The span of the diagnostic. */
+ Span span;
+
+ /*! \brief The line number. */
+ int line;
Review comment:
I am sorry! I am little confused here, may be i missed something.
Can you please help me understand, why we need line, column, end_line & end_column again, which we already maintain inside the Span object ?
The usual design i think would be ObjectBuilder --> Object .
##########
File path: python/tvm/parser/__init__.py
##########
@@ -24,4 +24,5 @@ def parse_expr(source):
return _ffi_api.ParseExpr("string", source)
def fromtext(source, source_name="from_string"):
+ # TODO(@tqchen): currently we have to invoke `str` which dramatically reduces performance.
Review comment:
Would you please help me understand this comment. Why you need to call str() if it is already a string object in python.
And if it is TVM::Runtime::String object, then much overloading is already supported to do the conversion.
Are you facing any error here?
##########
File path: include/tvm/parser/source_map.h
##########
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TVM_PARSER_SOURCE_MAP_H_
+#define TVM_PARSER_SOURCE_MAP_H_
+/*!
+ * \file source_map.h
+ * \brief A map from source names to source code.
+ */
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+#include <tvm/ir/span.h>
+
+#include <fstream>
+#include <string>
+
+namespace tvm {
+namespace parser {
+
+
+/*! \brief A program source in any language.
+ *
+ * Could represent the source from an ML framework or the internal
+ * source of a TVM program.
+ */
+struct Source {
+ /*! \brief The raw source. */
+ std::string source;
+ /*! \brief A mapping of line breaks into the raw source. */
+ std::vector<std::pair<int, int>> line_map;
+
+ /*! \brief An empty source. */
+ Source() : source(), line_map() {}
+
+ /*! \brief Construct a source from a string. */
+ TVM_DLL explicit Source(const std::string& source);
+
+ TVM_DLL Source(const Source& source) : source(source.source), line_map(source.line_map) {}
+
+ /*! \brief Generate an error message at a specific line and column with the
+ * annotated message.
+ *
+ * The error is written directly to the `out` std::ostream.
+ *
+ * \param out The output ostream.
+ * \param line The line at which to report a diagnostic.
+ * \param line The column at which to report a diagnostic.
+ * \param msg The message to attach.
+ */
+ TVM_DLL void ReportAt(std::ostream& out, int line, int column, const std::string& msg) const;
Review comment:
I think it would be good to use a TVM defined object to redirect the logs/debugs messages, rather than using ostream directly.
It helps to enable user to have a choice at later point in time, whether someone wants redirect logs to default screen or some files, or add some additional filters on top of it. And more importantly it is easily up-gradable.
Please let me know in case my point is not clear.
##########
File path: include/tvm/ir/span.h
##########
@@ -85,16 +85,26 @@ class SpanNode : public Object {
int line;
Review comment:
I believe what you are trying to achieve here is the range for line[begin, end], column[begin, end].
nit: May be we can rename the variables to reflect the same.
##########
File path: tests/python/relay/test_ir_parser.py
##########
@@ -868,48 +883,60 @@ def test_extern_adt_defn():
extern_def = relay.TypeData(extern_var, [typ_var], [])
mod[extern_var] = extern_def
- assert_parses_as(
+ assert_parse_module_as(
"""
extern type T[A]
""",
mod
)
+
+@pytest.mark.skip("not yet tested on parser 2.0")
def test_import_grad():
mod = tvm.IRModule()
mod.import_from_std("gradient.rly")
+# hiearchy id, i.e parse nn.conv2d
+# do with multiple levels
+#
+# call attributes not correctly parsing
+# convert error from attribute construction to real error message
+# lexing issue with projection of graph variables
+
+# def test_hierarchical_identifiers():
+# assert False
+
+def test_resnet():
+ mod, params = relay.testing.resnet.get_workload()
+ text = str(mod.astext())
+ parsed_mod = parse_module(text)
+ tvm.ir.assert_structural_equal(mod, parsed_mod)
+
+def inline_params(mod, params):
+ main_fn = mod["main"]
+ str_to_var = {}
+ for param in main_fn.params:
+ str_to_var[param.name_hint] = param
+
+ bind_map = {}
+ for param in params:
+ bind_map[str_to_var[param]] = relay.const(params[param])
+
+ body = relay.bind(main_fn.body, bind_map)
+ main_fn = relay.Function(relay.analysis.free_vars(body), body)
+ mod["main_fn"] = main_fn
+ return mod
+
+def test_resnet_inlined_params():
+ mod, params = relay.testing.resnet.get_workload()
+ print("here")
+ mod = inline_params(mod, params)
+ print("here")
+ text = str(mod.astext())
+ print("here")
+ parsed_mod = parse_module(text)
+ print("here")
+ tvm.ir.assert_structural_equal(mod, parsed_mod)
+ print("here")
+
if __name__ == "__main__":
- test_graph()
Review comment:
All test cases are removed ? If it is just a temp code, then this comment can act as reminder before merging :)
##########
File path: include/tvm/parser/source_map.h
##########
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TVM_PARSER_SOURCE_MAP_H_
+#define TVM_PARSER_SOURCE_MAP_H_
+/*!
+ * \file source_map.h
+ * \brief A map from source names to source code.
+ */
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+#include <tvm/ir/span.h>
+
+#include <fstream>
+#include <string>
+
+namespace tvm {
+namespace parser {
+
+
+/*! \brief A program source in any language.
+ *
+ * Could represent the source from an ML framework or the internal
+ * source of a TVM program.
+ */
+struct Source {
+ /*! \brief The raw source. */
+ std::string source;
+ /*! \brief A mapping of line breaks into the raw source. */
+ std::vector<std::pair<int, int>> line_map;
+
+ /*! \brief An empty source. */
+ Source() : source(), line_map() {}
+
+ /*! \brief Construct a source from a string. */
+ TVM_DLL explicit Source(const std::string& source);
+
+ TVM_DLL Source(const Source& source) : source(source.source), line_map(source.line_map) {}
+
+ /*! \brief Generate an error message at a specific line and column with the
+ * annotated message.
+ *
+ * The error is written directly to the `out` std::ostream.
+ *
+ * \param out The output ostream.
+ * \param line The line at which to report a diagnostic.
+ * \param line The column at which to report a diagnostic.
+ * \param msg The message to attach.
+ */
+ TVM_DLL void ReportAt(std::ostream& out, int line, int column, const std::string& msg) const;
+};
+
+/*!
+ * \brief A mapping from a unique source name to source fragment.
+ */
+class SourceMap;
Review comment:
nit: This class i think is more suitable if placed in span.h.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org