You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/06/05 16:06:52 UTC
[impala] 01/06: IMPALA-8564: Add table/view create time in the
lineage graph
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d9af99589fb9d8afd0844ee835b0fe945590e0bb
Author: Fredy Wijaya <fw...@cloudera.com>
AuthorDate: Tue May 21 18:01:21 2019 -0700
IMPALA-8564: Add table/view create time in the lineage graph
This patch adds table/view create time in the lineage graph. This is
needed for Impala/Atlas integration. See ATLAS-3080.
Below is an example of the updated lineage graph.
{
"queryText":"create table lineage_test_tbl as select int_col, tinyint_col from functional.alltypes",
"queryId":"0:0",
"hash":"407f23b24758ffcb2ac445b9703f5c44",
"user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
"sources":[
1
],
"targets":[
0
],
"edgeType":"PROJECTION"
},
{
"sources":[
3
],
"targets":[
2
],
"edgeType":"PROJECTION"
}
],
"vertices":[
{
"id":0,
"vertexType":"COLUMN",
"vertexId":"int_col",
"metadata":{
"tableName":"default.lineage_test_tbl",
"tableCreateTime":1559151337
}
},
{
"id":1,
"vertexType":"COLUMN",
"vertexId":"functional.alltypes.int_col",
"metadata":{
"tableName":"functional.alltypes",
"tableCreateTime":1559151317
}
},
{
"id":2,
"vertexType":"COLUMN",
"vertexId":"tinyint_col",
"metadata":{
"tableName":"default.lineage_test_tbl",
"tableCreateTime":1559151337
}
},
{
"id":3,
"vertexType":"COLUMN",
"vertexId":"functional.alltypes.tinyint_col",
"metadata":{
"tableName":"functional.alltypes",
"tableCreateTime":1559151317
}
}
]
}
Testing:
- Updated lineage tests in PlannerTest
- Updated test_lineage.py
- Ran all FE tests
Change-Id: If4f578d7b299a76c30323b10a883ba32f8713d82
Reviewed-on: http://gerrit.cloudera.org:8080/13399
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/client-request-state.h | 8 +-
be/src/service/frontend.cc | 3 +-
be/src/service/impala-server.cc | 16 +
be/src/util/lineage-util.h | 9 +
common/thrift/CatalogService.thrift | 8 +
common/thrift/LineageGraph.thrift | 15 +-
.../apache/impala/analysis/ColumnLineageGraph.java | 665 ++++---
.../impala/analysis/CreateOrAlterViewStmtBase.java | 5 +-
.../impala/catalog/CatalogServiceCatalog.java | 1 -
.../java/org/apache/impala/planner/Planner.java | 10 +-
.../apache/impala/service/CatalogOpExecutor.java | 7 +-
.../org/apache/impala/service/JniFrontend.java | 1 -
.../org/apache/impala/planner/PlannerTestBase.java | 8 +-
.../queries/PlannerTest/lineage.test | 2004 ++++++++++++++++----
tests/custom_cluster/test_lineage.py | 38 +-
15 files changed, 2181 insertions(+), 617 deletions(-)
diff --git a/be/src/service/client-request-state.h b/be/src/service/client-request-state.h
index ae3dc95..9347a08 100644
--- a/be/src/service/client-request-state.h
+++ b/be/src/service/client-request-state.h
@@ -197,7 +197,7 @@ class ClientRequestState {
void set_fetched_rows() { fetched_rows_ = true; }
bool fetched_rows() const { return fetched_rows_; }
bool returns_result_set() { return !result_metadata_.columns.empty(); }
- const TResultSetMetadata* result_metadata() { return &result_metadata_; }
+ const TResultSetMetadata* result_metadata() const { return &result_metadata_; }
const TUniqueId& query_id() const { return query_ctx_.query_id; }
const TExecRequest& exec_request() const { return exec_request_; }
TStmtType::type stmt_type() const { return exec_request_.stmt_type; }
@@ -259,7 +259,11 @@ class ClientRequestState {
RuntimeProfile::EventSequence* query_events() const { return query_events_; }
RuntimeProfile* summary_profile() { return summary_profile_; }
- protected:
+ /// Returns nullptr when catalog_op_type is not DDL.
+ const TDdlExecResponse* ddl_exec_response() const {
+ return catalog_op_executor_->ddl_exec_response();
+ }
+protected:
/// Updates the end_time_us_ of this query if it isn't set. The end time is determined
/// when this function is called for the first time, calling it multiple times does not
/// change the end time.
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index 6ef000a..757ef92 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -298,5 +298,4 @@ Status Frontend::BuildTestDescriptorTable(const TBuildTestDescriptorTableParams&
// Call FE post-query execution hook
Status Frontend::CallQueryCompleteHooks(const TQueryCompleteContext& context) {
return JniUtil::CallJniMethod(fe_, call_query_complete_hooks_id_, context);
-}
-
+}
\ No newline at end of file
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index eb6bec7..9fb58fe 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -480,6 +480,22 @@ Status ImpalaServer::LogLineageRecord(const ClientRequestState& client_request_s
} else {
return Status::OK();
}
+
+ if (client_request_state.catalog_op_type() == TCatalogOpType::DDL) {
+ const TDdlExecResponse* ddl_exec_response = client_request_state.ddl_exec_response();
+ // Update vertices that have -1 table_create_time for a newly created table/view.
+ if (ddl_exec_response->__isset.table_name &&
+ ddl_exec_response->__isset.table_create_time) {
+ for (auto &vertex: lineage_graph.vertices) {
+ if (!vertex.__isset.metadata) continue;
+ if (vertex.metadata.table_name == ddl_exec_response->table_name &&
+ vertex.metadata.table_create_time == -1) {
+ vertex.metadata.__set_table_create_time(ddl_exec_response->table_create_time);
+ }
+ }
+ }
+ }
+
// Set the query end time in TLineageGraph. Must use UNIX time directly rather than
// e.g. converting from client_request_state.end_time() (IMPALA-4440).
lineage_graph.__set_ended(UnixMillis() / 1000);
diff --git a/be/src/util/lineage-util.h b/be/src/util/lineage-util.h
index e52bb4c..f5dd722 100644
--- a/be/src/util/lineage-util.h
+++ b/be/src/util/lineage-util.h
@@ -42,6 +42,15 @@ class LineageUtil {
writer->String("COLUMN");
writer->String("vertexId");
writer->String(vertex.label.c_str());
+ if (vertex.__isset.metadata) {
+ writer->String("metadata");
+ writer->StartObject();
+ writer->String("tableName");
+ writer->String(vertex.metadata.table_name.c_str());
+ writer->String("tableCreateTime");
+ writer->Int64(vertex.metadata.table_create_time);
+ writer->EndObject();
+ }
writer->EndObject();
}
diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift
index f970d17..f76e2c3 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -164,6 +164,14 @@ struct TDdlExecResponse {
// Result of DDL operation to be returned to the client. Currently only set
// by COMPUTE STATS and ALTER TABLE.
3: optional Results.TResultSet result_set
+
+ // The table/view name in HMS. Set only for CREATE TABLE, CREATE TABLE AS SELECT,
+ // CREATE TABLE LIKE, and CREATE VIEW statements.
+ 4: optional string table_name
+
+ // The table/view create time stored in HMS. Set only for CREATE TABLE,
+ // CREATE TABLE AS SELECT, CREATE TABLE LIKE, and CREATE VIEW statements.
+ 5: optional i64 table_create_time
}
// Updates the metastore with new partition information and returns a response
diff --git a/common/thrift/LineageGraph.thrift b/common/thrift/LineageGraph.thrift
index b1d64b6..51e0493 100644
--- a/common/thrift/LineageGraph.thrift
+++ b/common/thrift/LineageGraph.thrift
@@ -20,17 +20,28 @@ namespace java org.apache.impala.thrift
include "Types.thrift"
+struct TVertexMetadata {
+ // Table name.
+ 1: required string table_name
+
+ // Create time of the table/view.
+ 2: required i64 table_create_time
+}
+
struct TVertex {
// Vertex id
1: required i64 id
// Column label
2: required string label
+
+ // Metadata of the vertex.
+ 3: optional TVertexMetadata metadata
}
enum TEdgeType {
- PROJECTION = 0
- PREDICATE = 1
+ PROJECTION = 0
+ PREDICATE = 1
}
struct TMultiEdge {
diff --git a/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java b/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java
index 53d3a3e..6285af6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java
@@ -21,12 +21,18 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
+import com.google.common.collect.ImmutableSortedMap;
+import org.apache.impala.analysis.ColumnLineageGraph.Vertex.Metadata;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.common.Id;
import org.apache.impala.common.IdGenerator;
@@ -36,6 +42,7 @@ import org.apache.impala.thrift.TMultiEdge;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TVertex;
+import org.apache.impala.thrift.TVertexMetadata;
import org.apache.impala.util.TUniqueIdUtil;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -54,238 +61,368 @@ import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
/**
- * Represents a vertex in the column lineage graph. A Vertex may correspond to a base
- * table column, a column in the destination table (for the case of INSERT or CTAS
- * queries) or a result expr (labeled column of a query result set).
+ * Represents the column lineage graph of a query. This is a directional graph that is
+ * used to track dependencies among the table/column entities that participate in
+ * a query. There are two types of dependencies that are represented as edges in the
+ * column lineage graph:
+ * a) Projection dependency: This is a dependency between a set of source
+ * columns (base table columns) and a single target (result expr or table column).
+ * This dependency indicates that values of the target depend on the values of the source
+ * columns.
+ * b) Predicate dependency: This is a dependency between a set of target
+ * columns (or exprs) and a set of source columns (base table columns). It indicates that
+ * the source columns restrict the values of their targets (e.g. by participating in
+ * WHERE clause predicates).
+ *
+ * The following dependencies are generated for a query:
+ * - Exactly one projection dependency for every result expr / target column.
+ * - Exactly one predicate dependency that targets all result exprs / target cols and
+ * depends on all columns participating in a conjunct in the query.
+ * - Special case of analytic fns: One predicate dependency per result expr / target col
+ * whose value is directly or indirectly affected by an analytic function with a
+ * partition by and/or order by clause.
*/
-final class Vertex implements Comparable<Vertex> {
- // Unique identifier of this vertex.
- private final VertexId id_;
-
- private final String type_ = "COLUMN";
-
- // A fully-qualified column name or the label of a result expr
- private final String label_;
-
- public Vertex(VertexId id, String label) {
- Preconditions.checkNotNull(id);
- Preconditions.checkNotNull(label);
- id_ = id;
- label_ = label;
- }
- public VertexId getVertexId() { return id_; }
- public String getLabel() { return label_; }
- public String getType() { return type_; }
-
- @Override
- public String toString() { return "(" + id_ + ":" + type_ + ":" + label_ + ")"; }
-
+public class ColumnLineageGraph {
/**
- * Encodes this Vertex object into a JSON object represented by a Map.
+ * Represents a vertex in the column lineage graph. A Vertex may correspond to a base
+ * table column, a column in the destination table (for the case of INSERT or CTAS
+ * queries) or a result expr (labeled column of a query result set).
*/
- public Map<String,Object> toJson() {
- // Use a LinkedHashMap to generate a strict ordering of elements.
- Map<String,Object> obj = new LinkedHashMap<>();
- obj.put("id", id_.asInt());
- obj.put("vertexType", type_);
- obj.put("vertexId", label_);
- return obj;
- }
+ public static final class Vertex implements Comparable<Vertex> {
+ public static class Metadata {
+ private final String tableName_; // the table name
+ private final long tableCreateTime_; // the table/view create time
+
+ public Metadata(String tableName, long tableCreateTime) {
+ tableName_ = tableName;
+ tableCreateTime_ = tableCreateTime;
+ }
- /**
- * Constructs a Vertex object from a JSON object. The new object is returned.
- */
- public static Vertex fromJsonObj(JSONObject obj) {
- int id = ((Long) obj.get("id")).intValue();
- String label = (String) obj.get("vertexId");
- return new Vertex(new VertexId(id), label);
- }
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Metadata metadata = (Metadata) o;
+ return tableCreateTime_ == metadata.tableCreateTime_ &&
+ Objects.equals(tableName_, metadata.tableName_);
+ }
- /**
- * Encodes this Vertex object into a thrift object
- */
- public TVertex toThrift() {
- return new TVertex(id_.asInt(), label_);
- }
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableName_, tableCreateTime_);
+ }
- /**
- * Constructs a Vertex object from a thrift object.
- */
- public static Vertex fromThrift(TVertex vertex) {
- int id = ((Long) vertex.id).intValue();
- return new Vertex(new VertexId(id), vertex.label);
- }
+ /**
+ * For testing only. We ignore the table create time.
+ */
+ private boolean equalsForTests(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Metadata metadata = (Metadata) o;
+ return Objects.equals(tableName_, metadata.tableName_);
+ }
+ }
- @Override
- public boolean equals(Object obj) {
- if (obj == null) return false;
- if (obj.getClass() != this.getClass()) return false;
- Vertex vertex = (Vertex) obj;
- return this.id_.equals(vertex.id_);
- }
+ // Unique identifier of this vertex.
+ private final VertexId id_;
+
+ private final String type_ = "COLUMN";
+
+ // A fully-qualified column name or the label of a result expr
+ private final String label_;
+
+ // The table metadata.
+ private final Metadata metadata_;
+
+ public Vertex(VertexId id, String label, Metadata metadata) {
+ Preconditions.checkNotNull(id);
+ Preconditions.checkNotNull(label);
+ id_ = id;
+ label_ = label;
+ metadata_ = metadata;
+ }
+ public VertexId getVertexId() { return id_; }
+ public String getLabel() { return label_; }
+ public String getType() { return type_; }
+ public Metadata getMetadata() { return metadata_; }
+
+ @Override
+ public String toString() { return "(" + id_ + ":" + type_ + ":" + label_ + ")"; }
+
+ /**
+ * Encodes this Vertex object into a JSON object represented by a Map.
+ */
+ public Map<String, Object> toJson() {
+ // Use a LinkedHashMap to generate a strict ordering of elements.
+ Map<String,Object> obj = new LinkedHashMap<>();
+ obj.put("id", id_.asInt());
+ obj.put("vertexType", type_);
+ obj.put("vertexId", label_);
+ if (metadata_ != null) {
+ JSONObject jsonMetadata = new JSONObject();
+ jsonMetadata.put("tableName", metadata_.tableName_);
+ jsonMetadata.put("tableCreateTime", metadata_.tableCreateTime_);
+ obj.put("metadata", jsonMetadata);
+ }
+ return obj;
+ }
+
+ /**
+ * Constructs a Vertex object from a JSON object. The new object is returned.
+ */
+ public static Vertex fromJsonObj(JSONObject obj) {
+ int id = ((Long) obj.get("id")).intValue();
+ String label = (String) obj.get("vertexId");
+ JSONObject jsonMetadata = (JSONObject) obj.get("metadata");
+ if (jsonMetadata == null) {
+ return new Vertex(new VertexId(id), label, null);
+ }
+ String tableName = (String) jsonMetadata.get("tableName");
+ long tableCreateTime = (Long) jsonMetadata.get("tableCreateTime");
+ return new Vertex(new VertexId(id), label, new Metadata(tableName,
+ tableCreateTime));
+ }
+
+ /**
+ * Encodes this Vertex object into a thrift object
+ */
+ public TVertex toThrift() {
+ TVertex vertex = new TVertex(id_.asInt(), label_);
+ if (metadata_ != null) {
+ TVertexMetadata metadata = new TVertexMetadata(metadata_.tableName_,
+ metadata_.tableCreateTime_);
+ vertex.setMetadata(metadata);
+ }
+ return vertex;
+ }
+
+ /**
+ * Constructs a Vertex object from a thrift object.
+ */
+ public static Vertex fromThrift(TVertex vertex) {
+ int id = ((Long) vertex.id).intValue();
+ TVertexMetadata thriftMetadata = vertex.getMetadata();
+ Metadata metadata = null;
+ if (thriftMetadata != null) {
+ metadata = new Metadata(thriftMetadata.getTable_name(),
+ thriftMetadata.getTable_create_time());
+ }
+ return new Vertex(new VertexId(id), vertex.label, metadata);
+ }
- @Override
- public int compareTo(Vertex cmp) { return this.id_.compareTo(cmp.id_); }
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Vertex vertex = (Vertex) o;
+ return Objects.equals(id_, vertex.id_) &&
+ Objects.equals(type_, vertex.type_) &&
+ Objects.equals(label_, vertex.label_) &&
+ Objects.equals(metadata_, vertex.metadata_);
+ }
- @Override
- public int hashCode() { return id_.hashCode(); }
-}
+ @Override
+ public int hashCode() {
+ return Objects.hash(id_, type_, label_, metadata_);
+ }
-/**
- * Represents the unique identifier of a Vertex.
- */
-class VertexId extends Id<VertexId> {
- protected VertexId(int id) {
- super(id);
- }
- public static IdGenerator<VertexId> createGenerator() {
- return new IdGenerator<VertexId>() {
- @Override
- public VertexId getNextId() { return new VertexId(nextId_++); }
- @Override
- public VertexId getMaxId() { return new VertexId(nextId_ - 1); }
- };
- }
-}
+ /**
+ * For testing only.
+ */
+ private boolean equalsForTests(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Vertex vertex = (Vertex) o;
+ return Objects.equals(id_, vertex.id_) &&
+ ((metadata_ == vertex.metadata_) ||
+ metadata_ != null && metadata_.equalsForTests(vertex.metadata_));
+ }
-/**
- * Represents a set of uni-directional edges in the column lineage graph, one edge from
- * every source Vertex in 'sources_' to every target Vertex in 'targets_'. An edge
- * indicates a dependency between a source and a target Vertex. There are two types of
- * edges, PROJECTION and PREDICATE, that are described in the ColumnLineageGraph class.
- */
-final class MultiEdge {
- public static enum EdgeType {
- PROJECTION, PREDICATE
- }
- private final Set<Vertex> sources_;
- private final Set<Vertex> targets_;
- private final EdgeType edgeType_;
-
- public MultiEdge(Set<Vertex> sources, Set<Vertex> targets, EdgeType type) {
- sources_ = sources;
- targets_ = targets;
- edgeType_ = type;
+ @Override
+ public int compareTo(Vertex cmp) { return this.id_.compareTo(cmp.id_); }
}
/**
- * Return an ordered set of source vertices.
+ * Represents the unique identifier of a Vertex.
*/
- private ImmutableSortedSet<Vertex> getOrderedSources() {
- return ImmutableSortedSet.copyOf(sources_);
+ private static final class VertexId extends Id<VertexId> {
+ protected VertexId(int id) {
+ super(id);
+ }
+ public static IdGenerator<VertexId> createGenerator() {
+ return new IdGenerator<VertexId>() {
+ @Override
+ public VertexId getNextId() { return new VertexId(nextId_++); }
+ @Override
+ public VertexId getMaxId() { return new VertexId(nextId_ - 1); }
+ };
+ }
}
/**
- * Return an ordered set of target vertices.
+ * Represents a set of uni-directional edges in the column lineage graph, one edge from
+ * every source Vertex in 'sources_' to every target Vertex in 'targets_'. An edge
+ * indicates a dependency between a source and a target Vertex. There are two types of
+ * edges, PROJECTION and PREDICATE, that are described in the ColumnLineageGraph class.
*/
- private ImmutableSortedSet<Vertex> getOrderedTargets() {
- return ImmutableSortedSet.copyOf(targets_);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- Joiner joiner = Joiner.on(",");
- builder.append("Sources: [");
- builder.append(joiner.join(getOrderedSources()) + "]\n");
- builder.append("Targets: [");
- builder.append(joiner.join(getOrderedTargets()) + "]\n");
- builder.append("Type: " + edgeType_);
- return builder.toString();
- }
+ private static final class MultiEdge {
+ public static enum EdgeType {
+ PROJECTION, PREDICATE
+ }
+ private final Set<Vertex> sources_;
+ private final Set<Vertex> targets_;
+ private final EdgeType edgeType_;
+
+ public MultiEdge(Set<Vertex> sources, Set<Vertex> targets, EdgeType type) {
+ sources_ = sources;
+ targets_ = targets;
+ edgeType_ = type;
+ }
+
+ /**
+ * Return an ordered set of source vertices.
+ */
+ private ImmutableSortedSet<Vertex> getOrderedSources() {
+ return ImmutableSortedSet.copyOf(sources_);
+ }
+
+ /**
+ * Return an ordered set of target vertices.
+ */
+ private ImmutableSortedSet<Vertex> getOrderedTargets() {
+ return ImmutableSortedSet.copyOf(targets_);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ Joiner joiner = Joiner.on(",");
+ builder.append("Sources: [");
+ builder.append(joiner.join(getOrderedSources()) + "]\n");
+ builder.append("Targets: [");
+ builder.append(joiner.join(getOrderedTargets()) + "]\n");
+ builder.append("Type: " + edgeType_);
+ return builder.toString();
+ }
+
+ /**
+ * Encodes this MultiEdge object to a JSON object represented by a Map.
+ * Returns a LinkedHashMap to guarantee a consistent ordering of elements.
+ */
+ public Map<String,Object> toJson() {
+ Map<String,Object> obj = new LinkedHashMap<>();
+ // Add sources
+ JSONArray sourceIds = new JSONArray();
+ for (Vertex vertex: getOrderedSources()) {
+ sourceIds.add(vertex.getVertexId());
+ }
+ obj.put("sources", sourceIds);
+ // Add targets
+ JSONArray targetIds = new JSONArray();
+ for (Vertex vertex: getOrderedTargets()) {
+ targetIds.add(vertex.getVertexId());
+ }
+ obj.put("targets", targetIds);
+ obj.put("edgeType", edgeType_.toString());
+ return obj;
+ }
+
+ /**
+ * Encodes this MultiEdge object to a thrift object
+ */
+ public TMultiEdge toThrift() {
+ List<TVertex> sources = new ArrayList<>();
+ for (Vertex vertex: getOrderedSources()) {
+ sources.add(vertex.toThrift());
+ }
+ List<TVertex> targets = new ArrayList<>();
+ for (Vertex vertex: getOrderedTargets()) {
+ targets.add(vertex.toThrift());
+ }
+ if (edgeType_ == EdgeType.PROJECTION) {
+ return new TMultiEdge(sources, targets, TEdgeType.PROJECTION);
+ }
+ return new TMultiEdge(sources, targets, TEdgeType.PREDICATE);
+ }
- /**
- * Encodes this MultiEdge object to a JSON object represented by a Map.
- * Returns a LinkedHashMap to guarantee a consistent ordering of elements.
- */
- public Map<String,Object> toJson() {
- Map<String,Object> obj = new LinkedHashMap<>();
- // Add sources
- JSONArray sourceIds = new JSONArray();
- for (Vertex vertex: getOrderedSources()) {
- sourceIds.add(vertex.getVertexId());
- }
- obj.put("sources", sourceIds);
- // Add targets
- JSONArray targetIds = new JSONArray();
- for (Vertex vertex: getOrderedTargets()) {
- targetIds.add(vertex.getVertexId());
- }
- obj.put("targets", targetIds);
- obj.put("edgeType", edgeType_.toString());
- return obj;
- }
+ /**
+ * Constructs a MultiEdge object from a thrift object.
+ */
+ public static MultiEdge fromThrift(TMultiEdge obj) {
+ Set<Vertex> sources = new HashSet<>();
+ for (TVertex vertex: obj.sources) {
+ sources.add(Vertex.fromThrift(vertex));
+ }
+ Set<Vertex> targets = new HashSet<>();
+ for (TVertex vertex: obj.targets) {
+ targets.add(Vertex.fromThrift(vertex));
+ }
+ if (obj.edgetype == TEdgeType.PROJECTION) {
+ return new MultiEdge(sources, targets, EdgeType.PROJECTION);
+ }
+ return new MultiEdge(sources, targets, EdgeType.PREDICATE);
+ }
- /**
- * Encodes this MultiEdge object to a thrift object
- */
- public TMultiEdge toThrift() {
- List<TVertex> sources = new ArrayList<>();
- for (Vertex vertex: getOrderedSources()) {
- sources.add(vertex.toThrift());
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ MultiEdge multiEdge = (MultiEdge) o;
+ return Objects.equals(sources_, multiEdge.sources_) &&
+ Objects.equals(targets_, multiEdge.targets_) &&
+ edgeType_ == multiEdge.edgeType_;
}
- List<TVertex> targets = new ArrayList<>();
- for (Vertex vertex: getOrderedTargets()) {
- targets.add(vertex.toThrift());
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(sources_, targets_, edgeType_);
}
- if (edgeType_ == EdgeType.PROJECTION) {
- return new TMultiEdge(sources, targets, TEdgeType.PROJECTION);
+
+ /**
+ * For testing only.
+ */
+ private boolean equalsForTests(Object obj) {
+ if (obj == null) return false;
+ if (obj.getClass() != this.getClass()) return false;
+ MultiEdge edge = (MultiEdge) obj;
+ return setEqualsForTests(edge.sources_, this.sources_) &&
+ setEqualsForTests(edge.targets_, this.targets_) &&
+ edge.edgeType_ == this.edgeType_;
}
- return new TMultiEdge(sources, targets, TEdgeType.PREDICATE);
}
- /**
- * Constructs a MultiEdge object from a thrift object.
- */
- public static MultiEdge fromThrift(TMultiEdge obj){
- Set<Vertex> sources = new HashSet<>();
- for (TVertex vertex: obj.sources) {
- sources.add(Vertex.fromThrift(vertex));
+ public static class ColumnLabel implements Comparable<ColumnLabel> {
+ private final String columnLabel_;
+ private final TableName tableName_;
+
+ public ColumnLabel(String columnName, TableName tableName) {
+ columnLabel_ = columnName;
+ tableName_ = tableName;
}
- Set<Vertex> targets = new HashSet<>();
- for (TVertex vertex: obj.targets) {
- targets.add(Vertex.fromThrift(vertex));
+
+ public ColumnLabel(String columnName) {
+ this(columnName, null);
}
- if (obj.edgetype == TEdgeType.PROJECTION) {
- return new MultiEdge(sources, targets, EdgeType.PROJECTION);
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ColumnLabel that = (ColumnLabel) o;
+ return Objects.equals(columnLabel_, that.columnLabel_);
}
- return new MultiEdge(sources, targets, EdgeType.PREDICATE);
- }
- @Override
- public boolean equals(Object obj) {
- if (obj == null) return false;
- if (obj.getClass() != this.getClass()) return false;
- MultiEdge edge = (MultiEdge) obj;
- return edge.sources_.equals(this.sources_) &&
- edge.targets_.equals(this.targets_) &&
- edge.edgeType_ == this.edgeType_;
+ @Override
+ public int hashCode() {
+ return Objects.hash(columnLabel_);
+ }
+
+ @Override
+ public int compareTo(ColumnLabel o) {
+ return columnLabel_.compareTo(o.columnLabel_);
+ }
}
-}
-/**
- * Represents the column lineage graph of a query. This is a directional graph that is
- * used to track dependencies among the table/column entities that participate in
- * a query. There are two types of dependencies that are represented as edges in the
- * column lineage graph:
- * a) Projection dependency: This is a dependency between a set of source
- * columns (base table columns) and a single target (result expr or table column).
- * This dependency indicates that values of the target depend on the values of the source
- * columns.
- * b) Predicate dependency: This is a dependency between a set of target
- * columns (or exprs) and a set of source columns (base table columns). It indicates that
- * the source columns restrict the values of their targets (e.g. by participating in
- * WHERE clause predicates).
- *
- * The following dependencies are generated for a query:
- * - Exactly one projection dependency for every result expr / target column.
- * - Exactly one predicate dependency that targets all result exprs / target cols and
- * depends on all columns participating in a conjunct in the query.
- * - Special case of analytic fns: One predicate dependency per result expr / target col
- * whose value is directly or indirectly affected by an analytic function with a
- * partition by and/or order by clause.
- */
-public class ColumnLineageGraph {
private final static Logger LOG = LoggerFactory.getLogger(ColumnLineageGraph.class);
// Query statement
private String queryStr_;
@@ -312,7 +449,7 @@ public class ColumnLineageGraph {
// For an INSERT or a CTAS, these are the columns of the
// destination table plus any partitioning columns (when dynamic partitioning is used).
// For a SELECT stmt, they are the labels of the result exprs.
- private final List<String> targetColumnLabels_ = new ArrayList<>();
+ private final List<ColumnLabel> targetColumnLabels_ = new ArrayList<>();
// Repository for tuple and slot descriptors for this query. Use it to construct the
// column lineage graph.
@@ -345,17 +482,35 @@ public class ColumnLineageGraph {
* 'targets' labels (representing column names or result expr labels). The new
* MultiEdge object is returned.
*/
- private MultiEdge createMultiEdge(Set<String> targets, Set<String> sources,
- MultiEdge.EdgeType type) {
+ private MultiEdge createMultiEdge(Set<ColumnLabel> targets,
+ Map<String, SlotDescriptor> sources, MultiEdge.EdgeType type, Analyzer analyzer) {
// createVertex() generates new IDs; we sort the input sets to make the output
// deterministic and independent of the ordering of the input sets.
Set<Vertex> targetVertices = new HashSet<>();
- for (String target: ImmutableSortedSet.copyOf(targets)) {
- targetVertices.add(createVertex(target));
+ for (ColumnLabel target: ImmutableSortedSet.copyOf(targets)) {
+ Metadata metadata = null;
+ if (target.tableName_ != null) {
+ FeTable feTable = analyzer.getStmtTableCache().tables.get(target.tableName_);
+ if (feTable != null && feTable.getMetaStoreTable() != null) {
+ metadata = new Metadata(target.tableName_.toString(),
+ feTable.getMetaStoreTable().getCreateTime());
+ } else {
+ // -1 is just a placeholder that will be updated after the table/view has been
+ // created. See impala-server.cc (LogLineageRecord) for more information.
+ metadata = new Metadata(target.tableName_.toString(), -1);
+ }
+ }
+ targetVertices.add(createVertex(target.columnLabel_, metadata));
}
Set<Vertex> sourceVertices = new HashSet<>();
- for (String source: ImmutableSortedSet.copyOf(sources)) {
- sourceVertices.add(createVertex(source));
+ for (Map.Entry<String, SlotDescriptor> source:
+ ImmutableSortedMap.copyOf(sources).entrySet()) {
+ FeTable feTable = source.getValue().getParent().getTable();
+ Preconditions.checkState(feTable != null);
+ Metadata metadata = feTable != null && feTable.getMetaStoreTable() != null ?
+ new Metadata(feTable.getTableName().toString(),
+ feTable.getMetaStoreTable().getCreateTime()) : null;
+ sourceVertices.add(createVertex(source.getKey(), metadata));
}
MultiEdge edge = new MultiEdge(sourceVertices, targetVertices, type);
edges_.add(edge);
@@ -366,10 +521,10 @@ public class ColumnLineageGraph {
* Creates a new vertex in the column lineage graph. The new Vertex object is
* returned. If a Vertex with the same label already exists, reuse it.
*/
- private Vertex createVertex(String label) {
+ private Vertex createVertex(String label, Metadata metadata) {
Vertex newVertex = vertices_.get(label);
if (newVertex != null) return newVertex;
- newVertex = new Vertex(vertexIdGenerator.getNextId(), label);
+ newVertex = new Vertex(vertexIdGenerator.getNextId(), label, metadata);
vertices_.put(newVertex.getLabel(), newVertex);
idToVertexMap_.put(newVertex.getVertexId(), newVertex);
return newVertex;
@@ -381,7 +536,7 @@ public class ColumnLineageGraph {
*/
public void computeLineageGraph(List<Expr> resultExprs, Analyzer rootAnalyzer) {
init(rootAnalyzer);
- computeProjectionDependencies(resultExprs);
+ computeProjectionDependencies(resultExprs, rootAnalyzer);
computeResultPredicateDependencies(rootAnalyzer);
}
@@ -404,28 +559,29 @@ public class ColumnLineageGraph {
queryId_ = queryCtx.query_id;
}
- private void computeProjectionDependencies(List<Expr> resultExprs) {
+ private void computeProjectionDependencies(List<Expr> resultExprs, Analyzer analyzer) {
Preconditions.checkNotNull(resultExprs);
Preconditions.checkState(!resultExprs.isEmpty());
Preconditions.checkState(resultExprs.size() == targetColumnLabels_.size());
for (int i = 0; i < resultExprs.size(); ++i) {
Expr expr = resultExprs.get(i);
- Set<String> sourceBaseCols = new HashSet<>();
+ Map<String, SlotDescriptor> sourceBaseCols = new HashMap<>();
List<Expr> dependentExprs = new ArrayList<>();
getSourceBaseCols(expr, sourceBaseCols, dependentExprs, false);
- Set<String> targets = Sets.newHashSet(targetColumnLabels_.get(i));
- createMultiEdge(targets, sourceBaseCols, MultiEdge.EdgeType.PROJECTION);
+ Set<ColumnLabel> targets = Sets.newHashSet(targetColumnLabels_.get(i));
+ createMultiEdge(targets, sourceBaseCols, MultiEdge.EdgeType.PROJECTION, analyzer);
if (!dependentExprs.isEmpty()) {
// We have additional exprs that 'expr' has a predicate dependency on.
// Gather the transitive predicate dependencies of 'expr' based on its direct
// predicate dependencies. For each direct predicate dependency p, 'expr' is
// transitively predicate dependent on all exprs that p is projection and
// predicate dependent on.
- Set<String> predicateBaseCols = new HashSet<>();
+ Map<String, SlotDescriptor> predicateBaseCols = new HashMap<>();
for (Expr dependentExpr: dependentExprs) {
getSourceBaseCols(dependentExpr, predicateBaseCols, null, true);
}
- createMultiEdge(targets, predicateBaseCols, MultiEdge.EdgeType.PREDICATE);
+ createMultiEdge(targets, predicateBaseCols, MultiEdge.EdgeType.PREDICATE,
+ analyzer);
}
}
}
@@ -441,13 +597,13 @@ public class ColumnLineageGraph {
if (expr.isAuxExpr()) continue;
resultDependencyPredicates_.add(expr);
}
- Set<String> predicateBaseCols = new HashSet<>();
+ Map<String, SlotDescriptor> predicateBaseCols = new HashMap<>();
for (Expr expr: resultDependencyPredicates_) {
getSourceBaseCols(expr, predicateBaseCols, null, true);
}
if (predicateBaseCols.isEmpty()) return;
- Set<String> targets = Sets.newHashSet(targetColumnLabels_);
- createMultiEdge(targets, predicateBaseCols, MultiEdge.EdgeType.PREDICATE);
+ Set<ColumnLabel> targets = Sets.newHashSet(targetColumnLabels_);
+ createMultiEdge(targets, predicateBaseCols, MultiEdge.EdgeType.PREDICATE, analyzer);
}
/**
@@ -460,7 +616,7 @@ public class ColumnLineageGraph {
* connected to. Which children are filtered depends on the type of 'expr' (e.g. for
* AnalyticFunctionExpr, grouping and sorting exprs are filtered out).
*/
- private void getSourceBaseCols(Expr expr, Set<String> sourceBaseCols,
+ private void getSourceBaseCols(Expr expr, Map<String, SlotDescriptor> sourceBaseCols,
List<Expr> directPredDeps, boolean traversePredDeps) {
List<Expr> exprsToTraverse = getProjectionDeps(expr);
List<Expr> predicateDepExprs = getPredicateDeps(expr);
@@ -478,7 +634,7 @@ public class ColumnLineageGraph {
// slot should correspond to a materialized tuple of a table
Preconditions.checkState(slotDesc.getParent().isMaterialized());
List<String> path = slotDesc.getPath().getCanonicalPath();
- sourceBaseCols.add(Joiner.on(".").join(path));
+ sourceBaseCols.put(Joiner.on(".").join(path), slotDesc);
} else {
for (Expr sourceExpr: sourceExprs) {
getSourceBaseCols(sourceExpr, sourceBaseCols, directPredDeps,
@@ -586,9 +742,11 @@ public class ColumnLineageGraph {
public static ColumnLineageGraph fromThrift(TLineageGraph obj) {
ColumnLineageGraph lineage =
new ColumnLineageGraph(obj.query_text, obj.query_id, obj.user, obj.started);
+ Map<TVertex, Vertex> vertexMap = new HashMap<>();
TreeSet<Vertex> vertices = Sets.newTreeSet();
for (TVertex vertex: obj.vertices) {
- vertices.add(Vertex.fromThrift(vertex));
+ Vertex v = Vertex.fromThrift(vertex);
+ vertices.add(v);
}
lineage.setVertices(vertices);
for (TMultiEdge edge: obj.edges) {
@@ -663,18 +821,68 @@ public class ColumnLineageGraph {
return vertices;
}
- @Override
- public boolean equals(Object obj) {
+ /**
+ * This is only for testing. It does not check for user and timestamp fields.
+ */
+ public boolean equalsForTests(Object obj) {
if (obj == null) return false;
if (obj.getClass() != this.getClass()) return false;
ColumnLineageGraph g = (ColumnLineageGraph) obj;
- if (!this.vertices_.equals(g.vertices_) ||
- !this.edges_.equals(g.edges_)) {
+ if (!mapEqualsForTests(this.vertices_, g.vertices_) ||
+ !listEqualsForTests(this.edges_, g.edges_)) {
return false;
}
return true;
}
+ private static boolean mapEqualsForTests(Map<String, Vertex> map1,
+ Map<String, Vertex> map2) {
+ if (map1.size() != map2.size()) return false;
+ Iterator<Entry<String, Vertex>> i = map1.entrySet().iterator();
+ while (i.hasNext()) {
+ Entry<String, Vertex> e = i.next();
+ String key = e.getKey();
+ Vertex value = e.getValue();
+ if (value == null) {
+ if (!(map2.get(key) == null && map2.containsKey(key))) return false;
+ } else {
+ if (!value.equalsForTests(map2.get(key))) return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean setEqualsForTests(Set<Vertex> set1, Set<Vertex> set2) {
+ if (set1.size() != set2.size()) return false;
+ for (Vertex v1 : set1) {
+ boolean found = false;
+ Iterator<Vertex> i = set2.iterator();
+ while (i.hasNext()) {
+ Vertex v2 = i.next();
+ if (v1.equalsForTests(v2)) {
+ i.remove();
+ found = true;
+ }
+ }
+ if (!found) return false;
+ }
+ return set2.isEmpty();
+ }
+
+ private static boolean listEqualsForTests(List<MultiEdge> list1,
+ List<MultiEdge> list2) {
+ ListIterator<MultiEdge> i1 = list1.listIterator();
+ ListIterator<MultiEdge> i2 = list2.listIterator();
+ while (i1.hasNext() && i2.hasNext()) {
+ MultiEdge e1 = i1.next();
+ MultiEdge e2 = i2.next();
+ if (!(e1 == null ? e2 == null : e1.equalsForTests(e2))) {
+ return false;
+ }
+ }
+ return !(i1.hasNext() || i2.hasNext());
+ }
+
public String debugString() {
StringBuilder builder = new StringBuilder();
for (MultiEdge edge: edges_) {
@@ -684,16 +892,15 @@ public class ColumnLineageGraph {
return builder.toString();
}
- public void addTargetColumnLabels(Collection<String> columnLabels) {
+ public void addTargetColumnLabels(Collection<ColumnLabel> columnLabels) {
Preconditions.checkNotNull(columnLabels);
targetColumnLabels_.addAll(columnLabels);
}
public void addTargetColumnLabels(FeTable dstTable) {
Preconditions.checkNotNull(dstTable);
- String tblFullName = dstTable.getFullName();
for (String columnName: dstTable.getColumnNames()) {
- targetColumnLabels_.add(tblFullName + "." + columnName);
+ targetColumnLabels_.add(new ColumnLabel(columnName, dstTable.getTableName()));
}
}
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java b/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
index e0ff1c9..5c064af 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.impala.analysis.ColumnLineageGraph.ColumnLabel;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TCreateOrAlterViewParams;
@@ -175,9 +176,9 @@ public abstract class CreateOrAlterViewStmtBase extends StatementBase {
*/
protected void computeLineageGraph(Analyzer analyzer) {
ColumnLineageGraph graph = analyzer.getColumnLineageGraph();
- List<String> colDefs = new ArrayList<>();
+ List<ColumnLabel> colDefs = new ArrayList<>();
for (ColumnDef colDef: finalColDefs_) {
- colDefs.add(dbName_ + "." + getTbl() + "." + colDef.getColName());
+ colDefs.add(new ColumnLabel(colDef.getColName(), new TableName(dbName_, getTbl())));
}
graph.addTargetColumnLabels(colDefs);
graph.computeLineageGraph(viewDefStmt_.getResultExprs(), analyzer);
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index 5f74c45..39a255d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.impala.analysis.TableName;
import org.apache.impala.authorization.AuthorizationDelta;
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index d4cc37b..087954b 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -20,11 +20,13 @@ package org.apache.impala.planner;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.impala.analysis.AnalysisContext;
import org.apache.impala.analysis.AnalysisContext.AnalysisResult;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.ColumnLineageGraph;
+import org.apache.impala.analysis.ColumnLineageGraph.ColumnLabel;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.ExprSubstitutionMap;
import org.apache.impala.analysis.InsertStmt;
@@ -192,10 +194,10 @@ public class Planner {
// the labels of columns mentioned in the column list.
List<String> mentionedColumns = insertStmt.getMentionedColumns();
Preconditions.checkState(!mentionedColumns.isEmpty());
- List<String> targetColLabels = new ArrayList<>();
+ List<ColumnLabel> targetColLabels = new ArrayList<>();
String tblFullName = targetTable.getFullName();
for (String column: mentionedColumns) {
- targetColLabels.add(tblFullName + "." + column);
+ targetColLabels.add(new ColumnLabel(column, targetTable.getTableName()));
}
graph.addTargetColumnLabels(targetColLabels);
} else {
@@ -213,7 +215,9 @@ public class Planner {
}
graph.computeLineageGraph(exprs, ctx_.getRootAnalyzer());
} else {
- graph.addTargetColumnLabels(ctx_.getQueryStmt().getColLabels());
+ graph.addTargetColumnLabels(ctx_.getQueryStmt().getColLabels().stream()
+ .map(col -> new ColumnLabel(col))
+ .collect(Collectors.toList()));
graph.computeLineageGraph(resultExprs, ctx_.getRootAnalyzer());
}
if (LOG.isTraceEnabled()) LOG.trace("lineage: " + graph.debugString());
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index e9cb876..1022e27 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -89,7 +89,6 @@ import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.catalog.TableNotFoundException;
import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.View;
-import org.apache.impala.catalog.events.MetastoreEvents;
import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEventPropertyKey;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
@@ -1982,7 +1981,13 @@ public class CatalogOpExecutor {
synchronized (metastoreDdlLock_) {
try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
msClient.getHiveClient().createTable(newTable);
+ // TODO (HIVE-21807): Creating a table and retrieving the table information is
+ // not atomic.
addSummary(response, "Table has been created.");
+ long tableCreateTime = msClient.getHiveClient().getTable(
+ newTable.getDbName(), newTable.getTableName()).getCreateTime();
+ response.setTable_name(newTable.getDbName() + "." + newTable.getTableName());
+ response.setTable_create_time(tableCreateTime);
// If this table should be cached, and the table location was not specified by
// the user, an extra step is needed to read the table to find the location.
if (cacheOp != null && cacheOp.isSet_cached() &&
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 631f9bd..139bb74 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -122,7 +122,6 @@ public class JniFrontend {
new TBinaryProtocol.Factory();
private final Frontend frontend_;
-
/**
* Create a new instance of the Jni Frontend.
*/
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
index 2917b08..3adb01f 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
@@ -740,13 +740,13 @@ public class PlannerTestBase extends FrontendTestBase {
ColumnLineageGraph outputGraph =
ColumnLineageGraph.fromThrift(lineageGraph);
if (expectedGraph == null || outputGraph == null ||
- !outputGraph.equals(expectedGraph)) {
+ !outputGraph.equalsForTests(expectedGraph)) {
StringBuilder lineageError = new StringBuilder();
lineageError.append("section " + Section.LINEAGE + " of query:\n"
+ query + "\n");
- lineageError.append("Output:\n");
- lineageError.append(outputGraph.toJson() + "\n");
- lineageError.append("Expected:\n");
+ lineageError.append("Output:");
+ lineageError.append(TestUtils.prettyPrintJson(outputGraph.toJson() + "\n"));
+ lineageError.append("\nExpected:\n");
lineageError.append(serializedGraph + "\n");
errorLog.append(lineageError.toString());
}
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test b/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
index 9e2e598..3d35c2f 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
@@ -9,7 +9,7 @@ select * from (
"queryText":"select * from (\n select tinyint_col + int_col x from functional.alltypes\n union all\n select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2",
"queryId":"0:0",
"hash":"25456c60a2e874a20732f42c7af27553",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -33,17 +33,29 @@ select * from (
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -63,7 +75,7 @@ order by b.bigint_col limit 10
"queryText":"select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),\n count(b.string_col), b.timestamp_col\nfrom functional.alltypes a join functional.alltypessmall b on (a.id = b.id)\nwhere a.year = 2010 and b.float_col > 0\ngroup by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col\nhaving count(a.int_col) > 10\norder by b.bigint_col limit 10",
"queryId":"0:0",
"hash":"e0309eeff9811f53c82657d62c1e04eb",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -133,17 +145,29 @@ order by b.bigint_col limit 10
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.smallint_col"
+ "vertexId":"functional.alltypes.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -153,7 +177,11 @@ order by b.bigint_col limit 10
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"functional.alltypessmall.string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":6,
@@ -163,32 +191,56 @@ order by b.bigint_col limit 10
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.timestamp_col"
+ "vertexId":"functional.alltypessmall.timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.year"
+ "vertexId":"functional.alltypes.year",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bigint_col"
+ "vertexId":"functional.alltypessmall.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"functional.alltypessmall.float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"functional.alltypessmall.id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -200,7 +252,7 @@ create table lineage_test_tbl as select int_col, tinyint_col from functional.all
"queryText":"create table lineage_test_tbl as select int_col, tinyint_col from functional.alltypes",
"queryId":"0:0",
"hash":"407f23b24758ffcb2ac445b9703f5c44",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -226,22 +278,38 @@ create table lineage_test_tbl as select int_col, tinyint_col from functional.all
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tbl.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tbl",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tbl.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tbl",
+ "tableCreateTime":-1
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -255,7 +323,7 @@ where a.year = 2009 and b.month = 2
"queryText":"create table lineage_test_tbl as\nselect distinct a.int_col, a.string_col from functional.alltypes a\ninner join functional.alltypessmall b on (a.id = b.id)\nwhere a.year = 2009 and b.month = 2",
"queryId":"0:0",
"hash":"f3101dcb046a7d34d7ee14892a6cc94e",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -294,42 +362,74 @@ where a.year = 2009 and b.month = 2
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tbl.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tbl",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tbl.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tbl",
+ "tableCreateTime":-1
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.year"
+ "vertexId":"functional.alltypes.year",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"functional.alltypessmall.id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.month"
+ "vertexId":"functional.alltypessmall.month",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -343,7 +443,7 @@ select * from
"queryText":"create table lineage_test_tbl as\nselect * from\n (select * from\n (select int_col from functional.alltypestiny limit 1) v1 ) v2",
"queryId":"0:0",
"hash":"9c04c1e9feee35ffacf14bfcd3b363a7",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -360,12 +460,20 @@ select * from
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tbl.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tbl",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.int_col"
+ "vertexId":"functional.alltypestiny.int_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
}
]
}
@@ -377,7 +485,7 @@ create table lineage_test_tblm as select * from functional_hbase.alltypes limit
"queryText":"create table lineage_test_tblm as select * from functional_hbase.alltypes limit 5",
"queryId":"0:0",
"hash":"a294f36bddf2adb329eac3055a76b2b5",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -502,132 +610,236 @@ create table lineage_test_tblm as select * from functional_hbase.alltypes limit
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.id"
+ "vertexId":"functional_hbase.alltypes.id",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.bigint_col"
+ "vertexId":"functional_hbase.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.bool_col"
+ "vertexId":"functional_hbase.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.date_string_col"
+ "vertexId":"functional_hbase.alltypes.date_string_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.double_col"
+ "vertexId":"functional_hbase.alltypes.double_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.float_col"
+ "vertexId":"functional_hbase.alltypes.float_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.int_col"
+ "vertexId":"functional_hbase.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.month"
+ "vertexId":"month",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.month"
+ "vertexId":"functional_hbase.alltypes.month",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":16,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":17,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.smallint_col"
+ "vertexId":"functional_hbase.alltypes.smallint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":18,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":19,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.string_col"
+ "vertexId":"functional_hbase.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":20,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":21,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.timestamp_col"
+ "vertexId":"functional_hbase.alltypes.timestamp_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":22,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":23,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.tinyint_col"
+ "vertexId":"functional_hbase.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":24,
"vertexType":"COLUMN",
- "vertexId":"default.lineage_test_tblm.year"
+ "vertexId":"year",
+ "metadata":{
+ "tableName":"default.lineage_test_tblm",
+ "tableCreateTime":-1
+ }
},
{
"id":25,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.year"
+ "vertexId":"functional_hbase.alltypes.year",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
}
]
}
@@ -642,7 +854,7 @@ functional_hbase.alltypes
"queryText":"insert into\nfunctional_hbase.alltypes\n values (1, 1, true, \"1999-12-01\", 2.0, 1.0, 1, 12, 2, \"abs\",\n cast(now() as timestamp), 1, 1999)",
"queryId":"0:0",
"hash":"b923425ce9cc2d53d36523ec83971e67",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -754,67 +966,119 @@ functional_hbase.alltypes
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.month"
+ "vertexId":"month",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypes.year"
+ "vertexId":"year",
+ "metadata":{
+ "tableName":"functional_hbase.alltypes",
+ "tableCreateTime":1559151687
+ }
}
]
}
@@ -827,7 +1091,7 @@ from functional.alltypes
"queryText":"insert into table functional.alltypesnopart (id, bool_col, timestamp_col)\nselect id, bool_col, timestamp_col\nfrom functional.alltypes",
"queryId":"0:0",
"hash":"b7b9474fc6b97f104bd031209438ee0e",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -926,72 +1190,128 @@ from functional.alltypes
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bool_col"
+ "vertexId":"functional.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesnopart.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypesnopart",
+ "tableCreateTime":1559151324
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.timestamp_col"
+ "vertexId":"functional.alltypes.timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -1006,7 +1326,7 @@ where year=2009 and month=05
"queryText":"insert into table functional.alltypessmall (smallint_col, int_col)\npartition (year=2009, month=04)\nselect smallint_col, int_col\nfrom functional.alltypes\nwhere year=2009 and month=05",
"queryId":"0:0",
"hash":"2ed3a6c784e1c0c7fcef226d71375180",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -1142,87 +1462,155 @@ where year=2009 and month=05
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.year"
+ "vertexId":"year",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.month"
+ "vertexId":"month",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.smallint_col"
+ "vertexId":"functional.alltypes.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.month"
+ "vertexId":"functional.alltypes.month",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":16,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.year"
+ "vertexId":"functional.alltypes.year",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -1237,7 +1625,7 @@ where year=2009 and month>10
"queryText":"insert into table functional.alltypessmall (id, string_col, int_col)\npartition (year, month)\nselect id, string_col, int_col, year, month\nfrom functional_seq_snap.alltypes\nwhere year=2009 and month>10",
"queryId":"0:0",
"hash":"39ac95ce0632ef1ee8b474be644971f3",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -1376,92 +1764,164 @@ where year=2009 and month>10
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.year"
+ "vertexId":"year",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional_seq_snap.alltypes.year"
+ "vertexId":"functional_seq_snap.alltypes.year",
+ "metadata":{
+ "tableName":"functional_seq_snap.alltypes",
+ "tableCreateTime":1559151137
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.month"
+ "vertexId":"month",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_seq_snap.alltypes.month"
+ "vertexId":"functional_seq_snap.alltypes.month",
+ "metadata":{
+ "tableName":"functional_seq_snap.alltypes",
+ "tableCreateTime":1559151137
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional_seq_snap.alltypes.id"
+ "vertexId":"functional_seq_snap.alltypes.id",
+ "metadata":{
+ "tableName":"functional_seq_snap.alltypes",
+ "tableCreateTime":1559151137
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional_seq_snap.alltypes.int_col"
+ "vertexId":"functional_seq_snap.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional_seq_snap.alltypes",
+ "tableCreateTime":1559151137
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":16,
"vertexType":"COLUMN",
- "vertexId":"functional_seq_snap.alltypes.string_col"
+ "vertexId":"functional_seq_snap.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional_seq_snap.alltypes",
+ "tableCreateTime":1559151137
+ }
},
{
"id":17,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -1480,7 +1940,7 @@ having min(id) > 10
"queryText":"insert into table functional.alltypessmall\npartition (year=2009, month)\nselect min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col),\nmin(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col),\nmin(timestamp_col), month\nfrom functional.alltypes\nwhere year=2009 and month>10\ngroup by month\nhaving min(id) > 10",
"queryId":"0:0",
"hash":"e6969c2cc67e9d6f3f985ddc6431f915",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -1627,132 +2087,236 @@ having min(id) > 10
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.year"
+ "vertexId":"year",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.month"
+ "vertexId":"month",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.month"
+ "vertexId":"functional.alltypes.month",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bool_col"
+ "vertexId":"functional.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.smallint_col"
+ "vertexId":"functional.alltypes.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":16,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.float_col"
+ "vertexId":"functional.alltypes.float_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":17,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":18,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.double_col"
+ "vertexId":"functional.alltypes.double_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":19,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":20,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.date_string_col"
+ "vertexId":"functional.alltypes.date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":21,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":22,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":23,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":24,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.timestamp_col"
+ "vertexId":"functional.alltypes.timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":25,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.year"
+ "vertexId":"functional.alltypes.year",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -1767,7 +2331,7 @@ group by int_col, tinyint_col
"queryText":"select\nmax(tinyint_col) over(partition by int_col)\nfrom functional.alltypes\ngroup by int_col, tinyint_col",
"queryId":"0:0",
"hash":"83c78528e6f5325c56a3f3521b08a78d",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -1798,12 +2362,20 @@ group by int_col, tinyint_col
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -1814,7 +2386,7 @@ select int_col, rank() over(order by int_col) from functional.alltypesagg
"queryText":"select int_col, rank() over(order by int_col) from functional.alltypesagg",
"queryId":"0:0",
"hash":"4f1ecaaed571d2ed9f09f091f399c311",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -1853,7 +2425,11 @@ select int_col, rank() over(order by int_col) from functional.alltypesagg
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.int_col"
+ "vertexId":"functional.alltypesagg.int_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":2,
@@ -1872,7 +2448,7 @@ order by a.tinyint_col, a.int_col
"queryText":"select a.tinyint_col, a.int_col, count(a.double_col)\n over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following)\nfrom functional.alltypes a inner join functional.alltypessmall b on a.id = b.id\norder by a.tinyint_col, a.int_col",
"queryId":"0:0",
"hash":"b6e26c00b2ef17f0592ebadb0ecc21f6",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -1934,7 +2510,11 @@ order by a.tinyint_col, a.int_col
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -1944,7 +2524,11 @@ order by a.tinyint_col, a.int_col
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -1954,17 +2538,29 @@ order by a.tinyint_col, a.int_col
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.double_col"
+ "vertexId":"functional.alltypes.double_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"functional.alltypessmall.id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -1983,7 +2579,7 @@ order by 2, 3, 4
"queryText":"with v2 as\n (select\n double_col,\n count(int_col) over() a,\n sum(int_col + bigint_col) over(partition by bool_col) b\n from\n (select * from functional.alltypes) v1)\nselect double_col, a, b, a + b, double_col + a from v2\norder by 2, 3, 4",
"queryId":"0:0",
"hash":"6bf993cea0d1ab9e613674ef178916c9",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -2062,7 +2658,11 @@ order by 2, 3, 4
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.double_col"
+ "vertexId":"functional.alltypes.double_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -2072,7 +2672,11 @@ order by 2, 3, 4
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -2082,12 +2686,20 @@ order by 2, 3, 4
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bool_col"
+ "vertexId":"functional.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":7,
@@ -2115,7 +2727,7 @@ order by 2, 3, 4
"queryText":"select double_col, a, b, a + b, double_col + a from\n (select\n double_col,\n count(int_col) over() a,\n sum(int_col + bigint_col) over(partition by bool_col) b\n from\n (select * from functional.alltypes) v1) v2\norder by 2, 3, 4",
"queryId":"0:0",
"hash":"811403c86e86fe630dea7bd0a6c89273",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867921,
"edges":[
{
@@ -2194,7 +2806,11 @@ order by 2, 3, 4
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.double_col"
+ "vertexId":"functional.alltypes.double_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -2204,7 +2820,11 @@ order by 2, 3, 4
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -2214,12 +2834,20 @@ order by 2, 3, 4
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bool_col"
+ "vertexId":"functional.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":7,
@@ -2249,7 +2877,7 @@ where b.month = 1
"queryText":"select a.month, a.year, b.int_col, b.month\nfrom\n (select year, month from functional.alltypes\n union all\n select year, month from functional.alltypes) a\n inner join\n functional.alltypessmall b\n on (a.month = b.month)\nwhere b.month = 1",
"queryId":"0:0",
"hash":"e3000cd5edf2a02e1f5407810f3cc09a",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2310,7 +2938,11 @@ where b.month = 1
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.month"
+ "vertexId":"functional.alltypes.month",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -2320,7 +2952,11 @@ where b.month = 1
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.year"
+ "vertexId":"functional.alltypes.year",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -2330,12 +2966,20 @@ where b.month = 1
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.int_col"
+ "vertexId":"functional.alltypessmall.int_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.month"
+ "vertexId":"functional.alltypessmall.month",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -2357,7 +3001,7 @@ where month = 1
"queryText":"select t1.int_col, t2.month, t2.int_col + 1\nfrom (\n select int_col, count(*)\n from functional.alltypessmall\n where month = 1\n group by int_col\n having count(*) > 1\n order by count(*) desc limit 5\n ) t1\njoin functional.alltypes t2 on (t1.int_col = t2.int_col)\nwhere month = 1",
"queryId":"0:0",
"hash":"3f1ecf7239e205342aee4979e7cb4877",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2411,7 +3055,11 @@ where month = 1
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.int_col"
+ "vertexId":"functional.alltypessmall.int_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":2,
@@ -2421,7 +3069,11 @@ where month = 1
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.month"
+ "vertexId":"functional.alltypes.month",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -2431,12 +3083,20 @@ where month = 1
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.month"
+ "vertexId":"functional.alltypessmall.month",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -2459,7 +3119,7 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
"queryText":"select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col\nfrom functional.alltypessmall c\njoin (\n select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day,\n a.int_col int_col, a.month month, b.float_col float_col, b.id id\n from ( select * from functional.alltypesagg a where month=1 ) a\n join functional.alltypessmall b on (a.smallint_col = b.id)\n ) x on (x.tinyint_col = c.id)\nwhere x.day=1\nand x.int_col [...]
"queryId":"0:0",
"hash":"4edf165aed5982ede63f7c91074f4b44",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2556,7 +3216,11 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.smallint_col"
+ "vertexId":"functional.alltypesagg.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":2,
@@ -2566,7 +3230,11 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"functional.alltypessmall.id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":4,
@@ -2576,7 +3244,11 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.tinyint_col"
+ "vertexId":"functional.alltypesagg.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":6,
@@ -2586,7 +3258,11 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.int_col"
+ "vertexId":"functional.alltypesagg.int_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":8,
@@ -2596,7 +3272,11 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"functional.alltypessmall.float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":10,
@@ -2606,17 +3286,29 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"functional.alltypessmall.string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.day"
+ "vertexId":"functional.alltypesagg.day",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.month"
+ "vertexId":"functional.alltypesagg.month",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
}
]
}
@@ -2636,7 +3328,7 @@ from
"queryText":"select c1, c2, c3\nfrom\n (select c1, c2, c3\n from\n (select int_col c1, sum(float_col) c2, min(float_col) c3\n from functional_hbase.alltypessmall\n group by 1) x\n order by 2,3 desc\n limit 5\n) y",
"queryId":"0:0",
"hash":"8b4d1ab11721d9ebdf26666d4195eb18",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2688,7 +3380,11 @@ from
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypessmall.int_col"
+ "vertexId":"functional_hbase.alltypessmall.int_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypessmall",
+ "tableCreateTime":1559151688
+ }
},
{
"id":2,
@@ -2698,7 +3394,11 @@ from
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypessmall.float_col"
+ "vertexId":"functional_hbase.alltypessmall.float_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypessmall",
+ "tableCreateTime":1559151688
+ }
},
{
"id":4,
@@ -2730,7 +3430,7 @@ limit 0
"queryText":"select c1, x2\nfrom (\n select c1, min(c2) x2\n from (\n select c1, c2, c3\n from (\n select int_col c1, tinyint_col c2, min(float_col) c3\n from functional_hbase.alltypessmall\n group by 1, 2\n order by 1,2\n limit 1\n ) x\n ) x2\n group by c1\n) y\norder by 2,1 desc\nlimit 0",
"queryId":"0:0",
"hash":"50d3b4f249f038b0711ea75c17640fc9",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2772,7 +3472,11 @@ limit 0
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypessmall.int_col"
+ "vertexId":"functional_hbase.alltypessmall.int_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypessmall",
+ "tableCreateTime":1559151688
+ }
},
{
"id":2,
@@ -2782,7 +3486,11 @@ limit 0
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypessmall.tinyint_col"
+ "vertexId":"functional_hbase.alltypessmall.tinyint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypessmall",
+ "tableCreateTime":1559151688
+ }
}
]
}
@@ -2794,7 +3502,7 @@ select int_col, string_col from functional.view_view
"queryText":"select int_col, string_col from functional.view_view",
"queryId":"0:0",
"hash":"9073496459077de1332e5017977dedf5",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2825,7 +3533,11 @@ select int_col, string_col from functional.view_view
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -2835,7 +3547,11 @@ select int_col, string_col from functional.view_view
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -2847,7 +3563,7 @@ where t.id < 10
"queryText":"select t.id from (select id from functional.alltypes_view) t\nwhere t.id < 10",
"queryId":"0:0",
"hash":"8ba7998033f90e1e358f4fdc7ea4251b",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2878,7 +3594,11 @@ where t.id < 10
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -2893,7 +3613,7 @@ where id in
"queryText":"select string_col, float_col, bool_col\nfrom functional.alltypes\nwhere id in\n (select id from functional.alltypesagg)",
"queryId":"0:0",
"hash":"e8ad1371d2a13e1ee9ec45689b62cdc9",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -2945,7 +3665,11 @@ where id in
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -2955,7 +3679,11 @@ where id in
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.float_col"
+ "vertexId":"functional.alltypes.float_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
@@ -2965,17 +3693,29 @@ where id in
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bool_col"
+ "vertexId":"functional.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.id"
+ "vertexId":"functional.alltypesagg.id",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
}
]
}
@@ -2993,7 +3733,7 @@ and tinyint_col < 10
"queryText":"select 1\nfrom functional.alltypesagg a\nwhere exists\n (select id, count(int_col) over (partition by bool_col)\n from functional.alltypestiny b\n where a.tinyint_col = b.tinyint_col\n group by id, int_col, bool_col)\nand tinyint_col < 10",
"queryId":"0:0",
"hash":"a7500c022d29c583c31b287868a848bf",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3024,12 +3764,20 @@ and tinyint_col < 10
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.tinyint_col"
+ "vertexId":"functional.alltypesagg.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.tinyint_col"
+ "vertexId":"functional.alltypestiny.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
}
]
}
@@ -3044,7 +3792,7 @@ and a.bigint_col > 10
"queryText":"select int_col + 1, tinyint_col - 1\nfrom functional.alltypes a\nwhere a.int_col <\n (select max(int_col) from functional.alltypesagg g where g.bool_col = true)\nand a.bigint_col > 10",
"queryId":"0:0",
"hash":"5e6227f323793ea4441e2a3119af2f09",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3088,7 +3836,11 @@ and a.bigint_col > 10
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -3098,22 +3850,38 @@ and a.bigint_col > 10
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.bool_col"
+ "vertexId":"functional.alltypesagg.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.int_col"
+ "vertexId":"functional.alltypesagg.int_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
}
]
}
@@ -3125,7 +3893,7 @@ with t as (select int_col x, bigint_col y from functional.alltypes) select x, y
"queryText":"with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t",
"queryId":"0:0",
"hash":"a7ab58d90540f28a8dfd69703632ad7a",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3156,7 +3924,11 @@ with t as (select int_col x, bigint_col y from functional.alltypes) select x, y
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
@@ -3166,7 +3938,11 @@ with t as (select int_col x, bigint_col y from functional.alltypes) select x, y
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -3179,7 +3955,7 @@ select id, int_col, string_col, year, month from t1
"queryText":"with t1 as (select * from functional.alltypestiny)\ninsert into functional.alltypesinsert (id, int_col, string_col) partition(year, month)\nselect id, int_col, string_col, year, month from t1",
"queryId":"0:0",
"hash":"0bc5b3e66cc72387f74893b1f1934946",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3296,92 +4072,164 @@ select id, int_col, string_col, year, month from t1
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.year"
+ "vertexId":"year",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.year"
+ "vertexId":"functional.alltypestiny.year",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.month"
+ "vertexId":"month",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.month"
+ "vertexId":"functional.alltypestiny.month",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.id"
+ "vertexId":"functional.alltypestiny.id",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.int_col"
+ "vertexId":"functional.alltypestiny.int_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
},
{
"id":16,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.string_col"
+ "vertexId":"functional.alltypestiny.string_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":17,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesinsert.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypesinsert",
+ "tableCreateTime":1559151331
+ }
}
]
}
@@ -3398,7 +4246,7 @@ from
"queryText":"select lead(a) over (partition by b order by c)\nfrom\n (select lead(id) over (partition by int_col order by bigint_col) as a,\n max(id) over (partition by tinyint_col order by int_col) as b,\n min(int_col) over (partition by string_col order by bool_col) as c\n from functional.alltypes) v",
"queryId":"0:0",
"hash":"aa95e5e6f39fc80bb3c318a2515dc77d",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3434,32 +4282,56 @@ from
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bigint_col"
+ "vertexId":"functional.alltypes.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.bool_col"
+ "vertexId":"functional.alltypes.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -3471,7 +4343,7 @@ create view test_view_lineage as select id from functional.alltypestiny
"queryText":"create view test_view_lineage as select id from functional.alltypestiny",
"queryId":"0:0",
"hash":"ff6b1ecb265afe4f03355a07238cfe37",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3488,12 +4360,20 @@ create view test_view_lineage as select id from functional.alltypestiny
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.id"
+ "vertexId":"functional.alltypestiny.id",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
}
]
}
@@ -3521,7 +4401,7 @@ limit 0
"queryText":"create view test_view_lineage (a, b) as select c1, x2\nfrom (\n select c1, min(c2) x2\n from (\n select c1, c2, c3\n from (\n select int_col c1, tinyint_col c2, min(float_col) c3\n from functional_hbase.alltypessmall\n group by 1, 2\n order by 1,2\n limit 1\n ) x\n ) x2\n group by c1\n) y\norder by 2,1 desc\nlimit 0",
"queryId":"0:0",
"hash":"b96adf892b897da1e562c5be98724fb5",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3558,22 +4438,38 @@ limit 0
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a"
+ "vertexId":"a",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypessmall.int_col"
+ "vertexId":"functional_hbase.alltypessmall.int_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypessmall",
+ "tableCreateTime":1559151688
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.b"
+ "vertexId":"b",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_hbase.alltypessmall.tinyint_col"
+ "vertexId":"functional_hbase.alltypessmall.tinyint_col",
+ "metadata":{
+ "tableName":"functional_hbase.alltypessmall",
+ "tableCreateTime":1559151688
+ }
}
]
}
@@ -3598,7 +4494,7 @@ create view test_view_lineage (a1, a2, a3, a4, a5, a6, a7) as
"queryText":"create view test_view_lineage (a1, a2, a3, a4, a5, a6, a7) as\n select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col\n from functional.alltypessmall c\n join (\n select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day,\n a.int_col int_col, a.month month, b.float_col float_col, b.id id\n from ( select * from functional.alltypesagg a where month=1 ) a\n join functional.alltypessmall b on (a.smallin [...]
"queryId":"0:0",
"hash":"ffbe643df8f26e92907fb45de1aeda36",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3691,77 +4587,137 @@ create view test_view_lineage (a1, a2, a3, a4, a5, a6, a7) as
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a1"
+ "vertexId":"a1",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.smallint_col"
+ "vertexId":"functional.alltypesagg.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a2"
+ "vertexId":"a2",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"functional.alltypessmall.id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a3"
+ "vertexId":"a3",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.tinyint_col"
+ "vertexId":"functional.alltypesagg.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a4"
+ "vertexId":"a4",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a5"
+ "vertexId":"a5",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.int_col"
+ "vertexId":"functional.alltypesagg.int_col",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a6"
+ "vertexId":"a6",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"functional.alltypessmall.float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.a7"
+ "vertexId":"a7",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"functional.alltypessmall.string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.day"
+ "vertexId":"functional.alltypesagg.day",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypesagg.month"
+ "vertexId":"functional.alltypesagg.month",
+ "metadata":{
+ "tableName":"functional.alltypesagg",
+ "tableCreateTime":1559151363
+ }
}
]
}
@@ -3781,7 +4737,7 @@ create view test_view_lineage as
"queryText":"create view test_view_lineage as\n select * from (\n select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),\n count(b.string_col), b.timestamp_col\n from functional.alltypes a join functional.alltypessmall b on (a.id = b.id)\n where a.year = 2010 and b.float_col > 0\n group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col\n having count(a.int_col) > 10\n order by b.bigint_col limit 10) t",
"queryId":"0:0",
"hash":"d4b9e2d63548088f911816b2ae29d7c2",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3846,67 +4802,119 @@ create view test_view_lineage as
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage._c0"
+ "vertexId":"_c0",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.tinyint_col"
+ "vertexId":"functional.alltypes.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.smallint_col"
+ "vertexId":"functional.alltypes.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage._c1"
+ "vertexId":"_c1",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.string_col"
+ "vertexId":"functional.alltypessmall.string_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.timestamp_col"
+ "vertexId":"timestamp_col",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.timestamp_col"
+ "vertexId":"functional.alltypessmall.timestamp_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.year"
+ "vertexId":"functional.alltypes.year",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.bigint_col"
+ "vertexId":"functional.alltypessmall.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.float_col"
+ "vertexId":"functional.alltypessmall.float_col",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypessmall.id"
+ "vertexId":"functional.alltypessmall.id",
+ "metadata":{
+ "tableName":"functional.alltypessmall",
+ "tableCreateTime":1559151325
+ }
}
]
}
@@ -3918,7 +4926,7 @@ alter view functional.alltypes_view as select id from functional.alltypestiny
"queryText":"alter view functional.alltypes_view as select id from functional.alltypestiny",
"queryId":"0:0",
"hash":"8c9367afc562a4c04d2d40e1276646c2",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3935,12 +4943,20 @@ alter view functional.alltypes_view as select id from functional.alltypestiny
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes_view.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional.alltypes_view",
+ "tableCreateTime":1559151386
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.id"
+ "vertexId":"functional.alltypestiny.id",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
}
]
}
@@ -3959,7 +4975,7 @@ select * from (
"queryText":"select * from (\n select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes\n where year = 2000\n order by nested_struct_col.f2.f12.f21 limit 10\n union all\n select sum(f1) y from\n (select complex_struct_col.f1 f1 from functional.allcomplextypes\n group by 1) v1) v2",
"queryId":"0:0",
"hash":"4fb3ceddbf596097335af607d528f5a7",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -3993,27 +5009,47 @@ select * from (
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.complex_struct_col.f1"
+ "vertexId":"functional.allcomplextypes.complex_struct_col.f1",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_struct_col.f1"
+ "vertexId":"functional.allcomplextypes.int_struct_col.f1",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_struct_col.f2"
+ "vertexId":"functional.allcomplextypes.int_struct_col.f2",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.nested_struct_col.f2.f12.f21"
+ "vertexId":"functional.allcomplextypes.nested_struct_col.f2.f12.f21",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.year"
+ "vertexId":"functional.allcomplextypes.year",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
}
]
}
@@ -4027,7 +5063,7 @@ select * from functional.allcomplextypes.int_array_col a inner join
"queryText":"select * from functional.allcomplextypes.int_array_col a inner join\n functional.allcomplextypes.struct_map_col m on (a.item = m.f1)",
"queryId":"0:0",
"hash":"8c0c64f8a4c08b82ad343ab439101957",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4089,7 +5125,11 @@ select * from functional.allcomplextypes.int_array_col a inner join
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_array_col.item"
+ "vertexId":"functional.allcomplextypes.int_array_col.item",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":2,
@@ -4099,7 +5139,11 @@ select * from functional.allcomplextypes.int_array_col a inner join
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.key"
+ "vertexId":"functional.allcomplextypes.struct_map_col.key",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":4,
@@ -4109,7 +5153,11 @@ select * from functional.allcomplextypes.int_array_col a inner join
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.value.f1"
+ "vertexId":"functional.allcomplextypes.struct_map_col.value.f1",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":6,
@@ -4119,7 +5167,11 @@ select * from functional.allcomplextypes.int_array_col a inner join
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.value.f2"
+ "vertexId":"functional.allcomplextypes.struct_map_col.value.f2",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
}
]
}
@@ -4133,7 +5185,7 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
"queryText":"select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m\n where a.item = m.f1",
"queryId":"0:0",
"hash":"1b0db371b32e90d33629ed7779332cf7",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4227,7 +5279,11 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.id"
+ "vertexId":"functional.allcomplextypes.id",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":2,
@@ -4237,7 +5293,11 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.year"
+ "vertexId":"functional.allcomplextypes.year",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":4,
@@ -4247,7 +5307,11 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.month"
+ "vertexId":"functional.allcomplextypes.month",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":6,
@@ -4257,7 +5321,11 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_array_col.item"
+ "vertexId":"functional.allcomplextypes.int_array_col.item",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":8,
@@ -4267,7 +5335,11 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.key"
+ "vertexId":"functional.allcomplextypes.struct_map_col.key",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":10,
@@ -4277,7 +5349,11 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.value.f1"
+ "vertexId":"functional.allcomplextypes.struct_map_col.value.f1",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":12,
@@ -4287,17 +5363,29 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.value.f2"
+ "vertexId":"functional.allcomplextypes.struct_map_col.value.f2",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_array_col"
+ "vertexId":"functional.allcomplextypes.int_array_col",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col"
+ "vertexId":"functional.allcomplextypes.struct_map_col",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
}
]
}
@@ -4318,7 +5406,7 @@ select a + b as ab, c, d, e from functional.allcomplextypes t,
"queryText":"select a + b as ab, c, d, e from functional.allcomplextypes t,\n (select sum(item) a from t.int_array_col\n where item < 10) v1,\n (select count(f1) b from t.struct_map_col\n group by key) v2,\n (select avg(value) over(partition by key) c from t.map_map_col.value) v3,\n (select item d from t.int_array_col\n union all\n select value from t.int_map_col) v4,\n (select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5",
"queryId":"0:0",
"hash":"4affc0d1e384475d1ff2fc2e19643064",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4392,12 +5480,20 @@ select a + b as ab, c, d, e from functional.allcomplextypes t,
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_array_col.item"
+ "vertexId":"functional.allcomplextypes.int_array_col.item",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.value.f1"
+ "vertexId":"functional.allcomplextypes.struct_map_col.value.f1",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":3,
@@ -4407,12 +5503,20 @@ select a + b as ab, c, d, e from functional.allcomplextypes t,
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.map_map_col.value.value"
+ "vertexId":"functional.allcomplextypes.map_map_col.value.value",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.map_map_col.value.key"
+ "vertexId":"functional.allcomplextypes.map_map_col.value.key",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":6,
@@ -4422,7 +5526,11 @@ select a + b as ab, c, d, e from functional.allcomplextypes t,
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.int_map_col.value"
+ "vertexId":"functional.allcomplextypes.int_map_col.value",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":8,
@@ -4432,17 +5540,29 @@ select a + b as ab, c, d, e from functional.allcomplextypes t,
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.value.f21"
+ "vertexId":"functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.value.f21",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.key"
+ "vertexId":"functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.key",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.allcomplextypes.struct_map_col.key"
+ "vertexId":"functional.allcomplextypes.struct_map_col.key",
+ "metadata":{
+ "tableName":"functional.allcomplextypes",
+ "tableCreateTime":1559151373
+ }
}
]
}
@@ -4456,7 +5576,7 @@ where not exists (select 1 from functional.alltypes a where v.id = a.id)
"queryText":"create view test_view_lineage as\nselect id from functional.alltypes_view v\nwhere not exists (select 1 from functional.alltypes a where v.id = a.id)",
"queryId":"0:0",
"hash":"e79b8abc8a682d9e0f6b2c30a6c885f3",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4482,12 +5602,20 @@ where not exists (select 1 from functional.alltypes a where v.id = a.id)
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.test_view_lineage.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"default.test_view_lineage",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -4500,7 +5628,7 @@ where k.int_col < 10
"queryText":"select count(*) from functional_kudu.alltypes k join functional.alltypes h on k.id = h.id\nwhere k.int_col < 10",
"queryId":"0:0",
"hash":"7b7c92d488186d869bb6b78c97666f41",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4532,17 +5660,29 @@ where k.int_col < 10
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.alltypes.id"
+ "vertexId":"functional_kudu.alltypes.id",
+ "metadata":{
+ "tableName":"functional_kudu.alltypes",
+ "tableCreateTime":1559150985
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.alltypes.int_col"
+ "vertexId":"functional_kudu.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional_kudu.alltypes",
+ "tableCreateTime":1559150985
+ }
}
]
}
@@ -4555,7 +5695,7 @@ functional.alltypes a where a.id < 100
"queryText":"insert into functional_kudu.testtbl select id, string_col as name, int_col as zip from\nfunctional.alltypes a where a.id < 100",
"queryId":"0:0",
"hash":"87a59bac56c6ad27f7af6e71af46d552",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4601,32 +5741,56 @@ functional.alltypes a where a.id < 100
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.name"
+ "vertexId":"name",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.zip"
+ "vertexId":"zip",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.int_col"
+ "vertexId":"functional.alltypes.int_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -4639,7 +5803,7 @@ functional.alltypes where id < 10
"queryText":"insert into functional_kudu.testtbl (name, id) select string_col as name, id from\nfunctional.alltypes where id < 10",
"queryId":"0:0",
"hash":"0bccfdbf4118e6d5a3d94062ecb5130a",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4675,22 +5839,38 @@ functional.alltypes where id < 10
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.name"
+ "vertexId":"name",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -4703,7 +5883,7 @@ functional.alltypes where id < 10
"queryText":"upsert into functional_kudu.testtbl (name, id) select string_col as name, id from\nfunctional.alltypes where id < 10",
"queryId":"0:0",
"hash":"f4c1e7b016e75012f7268f2f42ae5630",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4739,22 +5919,38 @@ functional.alltypes where id < 10
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.id"
+ "vertexId":"functional.alltypes.id",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"functional_kudu.testtbl.name"
+ "vertexId":"name",
+ "metadata":{
+ "tableName":"functional_kudu.testtbl",
+ "tableCreateTime":1559151047
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypes.string_col"
+ "vertexId":"functional.alltypes.string_col",
+ "metadata":{
+ "tableName":"functional.alltypes",
+ "tableCreateTime":1559151317
+ }
}
]
}
@@ -4769,7 +5965,7 @@ from functional.alltypestiny
"queryText":"create table kudu_ctas primary key (id) partition by hash (id) partitions 3\nstored as kudu as select id, bool_col, tinyint_col, smallint_col, int_col,\nbigint_col, float_col, double_col, date_string_col, string_col\nfrom functional.alltypestiny",
"queryId":"0:0",
"hash":"de98b09af6b6ab0f0678c5fc0c4369b4",
- "user":"progers",
+ "user":"dummy_user",
"timestamp":1547867922,
"edges":[
{
@@ -4867,103 +6063,183 @@ from functional.alltypestiny
{
"id":0,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.id"
+ "vertexId":"id",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":1,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.id"
+ "vertexId":"functional.alltypestiny.id",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":2,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.bool_col"
+ "vertexId":"bool_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":3,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.bool_col"
+ "vertexId":"functional.alltypestiny.bool_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":4,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.tinyint_col"
+ "vertexId":"tinyint_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":5,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.tinyint_col"
+ "vertexId":"functional.alltypestiny.tinyint_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":6,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.smallint_col"
+ "vertexId":"smallint_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":7,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.smallint_col"
+ "vertexId":"functional.alltypestiny.smallint_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":8,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.int_col"
+ "vertexId":"int_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":9,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.int_col"
+ "vertexId":"functional.alltypestiny.int_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":10,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.bigint_col"
+ "vertexId":"bigint_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":11,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.bigint_col"
+ "vertexId":"functional.alltypestiny.bigint_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":12,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.float_col"
+ "vertexId":"float_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":13,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.float_col"
+ "vertexId":"functional.alltypestiny.float_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":14,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.double_col"
+ "vertexId":"double_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":15,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.double_col"
+ "vertexId":"functional.alltypestiny.double_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":16,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.date_string_col"
+ "vertexId":"date_string_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":17,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.date_string_col"
+ "vertexId":"functional.alltypestiny.date_string_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
},
{
"id":18,
"vertexType":"COLUMN",
- "vertexId":"default.kudu_ctas.string_col"
+ "vertexId":"string_col",
+ "metadata":{
+ "tableName":"default.kudu_ctas",
+ "tableCreateTime":-1
+ }
},
{
"id":19,
"vertexType":"COLUMN",
- "vertexId":"functional.alltypestiny.string_col"
+ "vertexId":"functional.alltypestiny.string_col",
+ "metadata":{
+ "tableName":"functional.alltypestiny",
+ "tableCreateTime":1559151330
+ }
}
]
}
-====
+====
\ No newline at end of file
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 240f064..35469aa 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -32,14 +32,10 @@ from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
LOG = logging.getLogger(__name__)
-class TestLineage(CustomClusterTestSuite):
+class TestLineage(CustomClusterTestSuite):
lineage_log_dir = tempfile.mkdtemp()
- query = """
- select count(*) from functional.alltypes
- """
-
@classmethod
def setup_class(cls):
super(TestLineage, cls).setup_class()
@@ -55,7 +51,8 @@ class TestLineage(CustomClusterTestSuite):
UNIX times."""
LOG.info("lineage_event_log_dir is " + self.lineage_log_dir)
before_time = int(time.time())
- result = self.execute_query_expect_success(self.client, self.query)
+ query = "select count(*) from functional.alltypes"
+ result = self.execute_query_expect_success(self.client, query)
profile_query_id = re.search("Query \(id=(.*)\):", result.runtime_profile).group(1)
after_time = int(time.time())
LOG.info("before_time " + str(before_time) + " after_time " + str(after_time))
@@ -78,3 +75,32 @@ class TestLineage(CustomClusterTestSuite):
assert end_time <= after_time
else:
LOG.info("empty file: " + log_path)
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}".format(lineage_log_dir))
+ def test_create_table_timestamp(self, vector, unique_database):
+ """Test that 'createTableTime' in the lineage graph are populated with valid value
+ from HMS."""
+ query = "create table {0}.lineage_test_tbl as select int_col, tinyint_col " \
+ "from functional.alltypes".format(unique_database)
+ result = self.execute_query_expect_success(self.client, query)
+ profile_query_id = re.search("Query \(id=(.*)\):", result.runtime_profile).group(1)
+
+ # Wait to flush the lineage log files.
+ time.sleep(3)
+
+ for log_filename in os.listdir(self.lineage_log_dir):
+ log_path = os.path.join(self.lineage_log_dir, log_filename)
+ # Only the coordinator's log file will be populated.
+ if os.path.getsize(log_path) > 0:
+ with open(log_path) as log_file:
+ lineage_json = json.load(log_file)
+ assert lineage_json["queryId"] == profile_query_id
+ vertices = lineage_json["vertices"]
+ for vertex in vertices:
+ if vertex["vertexId"] == "int_col":
+ assert "metadata" in vertex
+ table_name = vertex["metadata"]["tableName"]
+ table_create_time = int(vertex["metadata"]["tableCreateTime"])
+ assert "{0}.lineage_test_tbl".format(unique_database) == table_name
+ assert table_create_time != -1