You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bh...@apache.org on 2019/09/12 03:48:15 UTC
[impala] branch master updated: IMPALA-8572: Log query events
before unregister.
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 32dbf5d IMPALA-8572: Log query events before unregister.
32dbf5d is described below
commit 32dbf5d78a4ced803a9182bfb0d7c0992497f97e
Author: Bharath Vissapragada <bh...@cloudera.com>
AuthorDate: Thu Aug 22 14:08:28 2019 -0700
IMPALA-8572: Log query events before unregister.
Currently, the query events (audits and lineages) are logged as a
part of query unregistration. This delays the event logging in cases
where the Unregister() is delayed by client for some reason (ex: Hue
does not call Unregister until the browser tab is closed) or the client
goes away without calling Unregister and the query timeout kicks in.
This patch moves this event logging to an earlier stage in the query
lifecycle. Moved the event logging related code into ClientRequestState
for easier code refactoring.
The conditions under which the events are logged are slightly
modified by this patch. Without the patch, events are logged for
unsuccessful queries if atleast a single fetch is perfomed. This patch
relaxes this guarantee to log events for any query that reaches
the FINISHED state (rows are available to fetch by the client) and does
not wait for a fetch to be performed. This simplifies the coordinator
state machine by avoiding unnecessary synchronization.
Added some test coverage for coordinator side code paths for writing
lineages. fe specific lineage tests only verified the correctness of
lineage created but did not test whether it was being flushed correctly
to the disk.
Change-Id: I639b9c1acb9806b29292cd85be2863688453ca2e
Reviewed-on: http://gerrit.cloudera.org:8080/14143
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/client-request-state.cc | 250 +-
be/src/service/client-request-state.h | 36 +-
be/src/service/impala-server.cc | 207 +-
be/src/service/impala-server.h | 21 +-
.../queries/QueryTest/lineage.test | 5648 ++++++++++++++++++++
tests/common/impala_test_suite.py | 51 +
tests/common/test_result_verifier.py | 22 +
tests/custom_cluster/test_lineage.py | 16 +
tests/unittests/test_file_parser.py | 8 +-
tests/util/test_file_parser.py | 2 +-
10 files changed, 6018 insertions(+), 243 deletions(-)
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index 64f63a8..e379eb7 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -21,6 +21,10 @@
#include <boost/algorithm/string/predicate.hpp>
#include <limits>
#include <gutil/strings/substitute.h>
+#include <rapidjson/rapidjson.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+#include <rapidjson/error/en.h>
#include "common/status.h"
#include "exec/kudu-util.h"
@@ -40,8 +44,11 @@
#include "service/query-result-set.h"
#include "util/debug-util.h"
#include "util/impalad-metrics.h"
+#include "util/lineage-util.h"
#include "util/metrics.h"
#include "util/pretty-printer.h"
+#include "util/promise.h"
+#include "util/redactor.h"
#include "util/runtime-profile-counters.h"
#include "util/time.h"
@@ -57,12 +64,15 @@
using boost::algorithm::iequals;
using boost::algorithm::join;
+using boost::algorithm::replace_all_copy;
using kudu::rpc::RpcController;
using namespace apache::hive::service::cli::thrift;
using namespace apache::thrift;
using namespace beeswax;
using namespace strings;
+DECLARE_bool(abort_on_failed_audit_event);
+DECLARE_bool(abort_on_failed_lineage_event);
DECLARE_int32(krpc_port);
DECLARE_int32(catalog_service_port);
DECLARE_string(catalog_service_host);
@@ -727,11 +737,24 @@ void ClientRequestState::Done() {
if (InTransaction()) AbortTransaction();
UpdateEndTime();
- unique_lock<mutex> l(lock_);
- query_events_->MarkEvent("Unregister query");
- // Update result set cache metrics, and update mem limit accounting before tearing
- // down the coordinator.
- ClearResultCache();
+
+ {
+ unique_lock<mutex> l(lock_);
+ query_events_->MarkEvent("Unregister query");
+ // Update result set cache metrics, and update mem limit accounting before tearing
+ // down the coordinator.
+ ClearResultCache();
+ }
+ // Wait until the audit events are flushed.
+ if (wait_thread_.get() != nullptr) {
+ wait_thread_->Join();
+ wait_thread_.reset();
+ } else {
+ // The query failed in the fe even before a wait thread is launched. Synchronously
+ // flush log events to audit authorization errors, if any.
+ LogQueryEvents();
+ }
+ DCHECK(wait_thread_.get() == nullptr);
}
Status ClientRequestState::Exec(const TMetadataOpRequest& exec_request) {
@@ -754,24 +777,10 @@ Status ClientRequestState::WaitAsync() {
void ClientRequestState::BlockOnWait() {
unique_lock<mutex> l(lock_);
- if (wait_thread_.get() == NULL) return;
- if (!is_block_on_wait_joining_) {
- // No other thread is already joining on wait_thread_, so this thread needs to do
- // it. Other threads will need to block on the cond-var.
- is_block_on_wait_joining_ = true;
- l.unlock();
- wait_thread_->Join();
- l.lock();
- is_block_on_wait_joining_ = false;
- wait_thread_.reset();
- block_on_wait_cv_.NotifyAll();
- } else {
- // Another thread is already joining with wait_thread_. Block on the cond-var
- // until the Join() executed in the other thread has completed.
- do {
- block_on_wait_cv_.Wait(l);
- } while (is_block_on_wait_joining_);
- }
+ // Some metadata operations like GET_COLUMNS do not rely on WaitAsync() to launch
+ // the wait thread. In such cases this method is expected to be a no-op.
+ if (wait_thread_.get() == nullptr) return;
+ while (!is_wait_done_) block_on_wait_cv_.Wait(l);
}
void ClientRequestState::Wait() {
@@ -786,6 +795,7 @@ void ClientRequestState::Wait() {
}
discard_result(UpdateQueryStatus(status));
}
+
if (status.ok()) {
if (stmt_type() == TStmtType::DDL) {
DCHECK(catalog_op_type() != TCatalogOpType::DDL || request_result_set_ != nullptr);
@@ -795,6 +805,14 @@ void ClientRequestState::Wait() {
// UpdateQueryStatus() or UpdateNonErrorOperationState() have updated operation_state_.
DCHECK(operation_state_ == TOperationState::FINISHED_STATE ||
operation_state_ == TOperationState::ERROR_STATE);
+ // Notify all the threads blocked on Wait() to finish and then log the query events,
+ // if any.
+ {
+ unique_lock<mutex> l(lock_);
+ is_wait_done_ = true;
+ }
+ block_on_wait_cv_.NotifyAll();
+ LogQueryEvents();
}
Status ClientRequestState::WaitInternal() {
@@ -1372,4 +1390,190 @@ void ClientRequestState::ClearTransactionState() {
transaction_closed_ = true;
}
+void ClientRequestState::LogQueryEvents() {
+ // Wait until the results are available. This guarantees the completion of non QUERY
+ // statemens like DDL/DML etc. Query events are logged if the query reaches a FINISHED
+ // state. For certain query types, events are logged regardless of the query state.
+ Status status;
+ {
+ lock_guard<mutex> l(lock_);
+ status = query_status();
+ }
+ bool log_events = true;
+ switch (stmt_type()) {
+ case TStmtType::QUERY:
+ case TStmtType::DML:
+ case TStmtType::DDL:
+ log_events = status.ok();
+ break;
+ case TStmtType::EXPLAIN:
+ case TStmtType::LOAD:
+ case TStmtType::SET:
+ case TStmtType::ADMIN_FN:
+ default:
+ break;
+ }
+
+ // Log audit events that are due to an AuthorizationException.
+ if (parent_server_->IsAuditEventLoggingEnabled() &&
+ (Frontend::IsAuthorizationError(status) || log_events)) {
+ // TODO: deal with an error status
+ discard_result(LogAuditRecord(status));
+ }
+
+ if (log_events && (parent_server_->AreQueryHooksEnabled() ||
+ parent_server_->IsLineageLoggingEnabled())) {
+ // TODO: deal with an error status
+ discard_result(LogLineageRecord());
+ }
+}
+
+Status ClientRequestState::LogAuditRecord(const Status& query_status) {
+ const TExecRequest& request = exec_request();
+ stringstream ss;
+ rapidjson::StringBuffer buffer;
+ rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+
+ writer.StartObject();
+ // Each log entry is a timestamp mapped to a JSON object
+ ss << UnixMillis();
+ writer.String(ss.str().c_str());
+ writer.StartObject();
+ writer.String("query_id");
+ writer.String(PrintId(query_id()).c_str());
+ writer.String("session_id");
+ writer.String(PrintId(session_id()).c_str());
+ writer.String("start_time");
+ writer.String(ToStringFromUnixMicros(start_time_us()).c_str());
+ writer.String("authorization_failure");
+ writer.Bool(Frontend::IsAuthorizationError(query_status));
+ writer.String("status");
+ writer.String(query_status.GetDetail().c_str());
+ writer.String("user");
+ writer.String(effective_user().c_str());
+ writer.String("impersonator");
+ if (do_as_user().empty()) {
+ // If there is no do_as_user() is empty, the "impersonator" field should be Null.
+ writer.Null();
+ } else {
+ // Otherwise, the delegator is the current connected user.
+ writer.String(connected_user().c_str());
+ }
+ writer.String("statement_type");
+ if (request.stmt_type == TStmtType::DDL) {
+ if (request.catalog_op_request.op_type == TCatalogOpType::DDL) {
+ writer.String(
+ PrintThriftEnum(request.catalog_op_request.ddl_params.ddl_type).c_str());
+ } else {
+ writer.String(PrintThriftEnum(request.catalog_op_request.op_type).c_str());
+ }
+ } else {
+ writer.String(PrintThriftEnum(request.stmt_type).c_str());
+ }
+ writer.String("network_address");
+ writer.String(TNetworkAddressToString(
+ session()->network_address).c_str());
+ writer.String("sql_statement");
+ string stmt = replace_all_copy(sql_stmt(), "\n", " ");
+ Redact(&stmt);
+ writer.String(stmt.c_str());
+ writer.String("catalog_objects");
+
+ writer.StartArray();
+ for (const TAccessEvent& event: request.access_events) {
+ writer.StartObject();
+ writer.String("name");
+ writer.String(event.name.c_str());
+ writer.String("object_type");
+ writer.String(PrintThriftEnum(event.object_type).c_str());
+ writer.String("privilege");
+ writer.String(event.privilege.c_str());
+ writer.EndObject();
+ }
+ writer.EndArray();
+ writer.EndObject();
+ writer.EndObject();
+ Status status = parent_server_->AppendAuditEntry(buffer.GetString());
+ if (!status.ok()) {
+ LOG(ERROR) << "Unable to record audit event record: " << status.GetDetail();
+ if (FLAGS_abort_on_failed_audit_event) {
+ CLEAN_EXIT_WITH_ERROR("Shutting down Impala Server due to "
+ "abort_on_failed_audit_event=true");
+ }
+ }
+ return status;
+}
+
+Status ClientRequestState::LogLineageRecord() {
+ const TExecRequest& request = exec_request();
+ if (!request.__isset.query_exec_request && !request.__isset.catalog_op_request) {
+ return Status::OK();
+ }
+ TLineageGraph lineage_graph;
+ if (request.__isset.query_exec_request &&
+ request.query_exec_request.__isset.lineage_graph) {
+ lineage_graph = request.query_exec_request.lineage_graph;
+ } else if (request.__isset.catalog_op_request &&
+ request.catalog_op_request.__isset.lineage_graph) {
+ lineage_graph = request.catalog_op_request.lineage_graph;
+ } else {
+ return Status::OK();
+ }
+
+ if (catalog_op_type() == TCatalogOpType::DDL) {
+ const TDdlExecResponse* response = ddl_exec_response();
+ // Update vertices that have -1 table_create_time for a newly created table/view.
+ if (response->__isset.table_name &&
+ response->__isset.table_create_time) {
+ for (auto &vertex: lineage_graph.vertices) {
+ if (!vertex.__isset.metadata) continue;
+ if (vertex.metadata.table_name == response->table_name &&
+ vertex.metadata.table_create_time == -1) {
+ vertex.metadata.__set_table_create_time(response->table_create_time);
+ }
+ }
+ }
+ }
+
+ // Set the query end time in TLineageGraph. Must use UNIX time directly rather than
+ // e.g. converting from end_time() (IMPALA-4440).
+ lineage_graph.__set_ended(UnixMillis() / 1000);
+
+ string lineage_record;
+ LineageUtil::TLineageToJSON(lineage_graph, &lineage_record);
+
+ if (parent_server_->AreQueryHooksEnabled()) {
+ // invoke QueryEventHooks
+ TQueryCompleteContext query_complete_context;
+ query_complete_context.__set_lineage_string(lineage_record);
+ const Status& status = exec_env_->frontend()->CallQueryCompleteHooks(
+ query_complete_context);
+
+ if (!status.ok()) {
+ LOG(ERROR) << "Failed to send query lineage info to FE CallQueryCompleteHooks"
+ << status.GetDetail();
+ if (FLAGS_abort_on_failed_lineage_event) {
+ CLEAN_EXIT_WITH_ERROR("Shutting down Impala Server due to "
+ "abort_on_failed_lineage_event=true");
+ }
+ }
+ }
+
+ // lineage logfile writing is deprecated in favor of the
+ // QueryEventHooks (see FE). this behavior is being retained
+ // for now but may be removed in the future.
+ if (parent_server_->IsLineageLoggingEnabled()) {
+ const Status& status = parent_server_->AppendLineageEntry(lineage_record);
+ if (!status.ok()) {
+ LOG(ERROR) << "Unable to record query lineage record: " << status.GetDetail();
+ if (FLAGS_abort_on_failed_lineage_event) {
+ CLEAN_EXIT_WITH_ERROR("Shutting down Impala Server due to "
+ "abort_on_failed_lineage_event=true");
+ }
+ }
+ return status;
+ }
+ return Status::OK();
+}
+
}
diff --git a/be/src/service/client-request-state.h b/be/src/service/client-request-state.h
index 4f1d712..475a4ce 100644
--- a/be/src/service/client-request-state.h
+++ b/be/src/service/client-request-state.h
@@ -88,16 +88,24 @@ class ClientRequestState {
/// Call this to ensure that rows are ready when calling FetchRows(). Updates the
/// query_status_, and advances operation_state_ to FINISHED or EXCEPTION. Must be
/// preceded by call to Exec(). Waits for all child queries to complete. Takes lock_.
+ /// Since this is a blocking operation, it is invoked from a separate thread
+ /// (WaitAsync()) and all the callers that intend to block until Wait() completes
+ /// are expected to use BlockOnWait().
+ ///
+ /// Logs the query events once the query reaches the FINISHED state. Threads blocked
+ /// on Wait() are signalled *before* logging the events so that they can resume their
+ /// execution (like fetching the results) and are not blocked until the event logging
+ /// is complete.
void Wait();
/// Calls Wait() asynchronously in a thread and returns immediately.
Status WaitAsync();
/// BlockOnWait() may be called after WaitAsync() has been called in order to wait
- /// for the asynchronous thread (wait_thread_) to complete. It is safe to call this
- /// from multiple threads (all threads will block until wait_thread_ has completed)
- /// and multiple times (non-blocking once wait_thread_ has completed). Do not call
- /// while holding lock_.
+ /// for the asynchronous thread (wait_thread_) to signal block_on_wait_cv_. It is
+ /// thread-safe and all the caller threads will block until wait_thread_ has
+ /// completed) and multiple times (non-blocking once wait_thread_ has completed).
+ /// Do not call while holding lock_.
void BlockOnWait();
/// Return at most max_rows from the current batch. If the entire current batch has
@@ -330,12 +338,12 @@ protected:
/// Thread for asynchronously running Wait().
std::unique_ptr<Thread> wait_thread_;
- /// Condition variable to make BlockOnWait() thread-safe. One thread joins
- /// wait_thread_, and all other threads block on this cv. Used with lock_.
+ /// Condition variable used to signal the threads that are blocked on Wait() to finish.
+ /// Callers are expected to use BlockOnWait() for Wait() to finish.
ConditionVariable block_on_wait_cv_;
- /// Used in conjunction with block_on_wait_cv_ to make BlockOnWait() thread-safe.
- bool is_block_on_wait_joining_ = false;
+ /// Wait condition used in conjunction with block_on_wait_cv_.
+ bool is_wait_done_ = false;
/// Session that this query is from
std::shared_ptr<ImpalaServer::SessionState> session_;
@@ -553,6 +561,18 @@ protected:
/// Invoke this function when the transaction is committed or aborted.
void ClearTransactionState();
+
+ /// helper that logs the audit record for this query id. Takes the query_status
+ /// as input parameter so that it operates on the same status polled in the
+ /// beginning of LogQueryEvents().
+ Status LogAuditRecord(const Status& query_status) WARN_UNUSED_RESULT;
+
+ /// Helper that logs the lineage record for this query id.
+ Status LogLineageRecord() WARN_UNUSED_RESULT;
+
+ /// Logs audit and column lineage events. Expects that Wait() has already finished.
+ /// Grabs lock_ for polling the query_status(). Hence do not call it under lock_.
+ void LogQueryEvents();
};
}
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index b3083ae..b6abb0a 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -77,7 +77,6 @@
#include "util/error-util.h"
#include "util/histogram-metric.h"
#include "util/impalad-metrics.h"
-#include "util/lineage-util.h"
#include "util/metrics.h"
#include "util/network-util.h"
#include "util/openssl-util.h"
@@ -521,78 +520,6 @@ Status ImpalaServer::PopulateAuthorizedProxyConfig(
return Status::OK();
}
-Status ImpalaServer::LogLineageRecord(const ClientRequestState& client_request_state) {
- const TExecRequest& request = client_request_state.exec_request();
- if (!request.__isset.query_exec_request && !request.__isset.catalog_op_request) {
- return Status::OK();
- }
- TLineageGraph lineage_graph;
- if (request.__isset.query_exec_request &&
- request.query_exec_request.__isset.lineage_graph) {
- lineage_graph = request.query_exec_request.lineage_graph;
- } else if (request.__isset.catalog_op_request &&
- request.catalog_op_request.__isset.lineage_graph) {
- lineage_graph = request.catalog_op_request.lineage_graph;
- } else {
- return Status::OK();
- }
-
- if (client_request_state.catalog_op_type() == TCatalogOpType::DDL) {
- const TDdlExecResponse* ddl_exec_response = client_request_state.ddl_exec_response();
- // Update vertices that have -1 table_create_time for a newly created table/view.
- if (ddl_exec_response->__isset.table_name &&
- ddl_exec_response->__isset.table_create_time) {
- for (auto &vertex: lineage_graph.vertices) {
- if (!vertex.__isset.metadata) continue;
- if (vertex.metadata.table_name == ddl_exec_response->table_name &&
- vertex.metadata.table_create_time == -1) {
- vertex.metadata.__set_table_create_time(ddl_exec_response->table_create_time);
- }
- }
- }
- }
-
- // Set the query end time in TLineageGraph. Must use UNIX time directly rather than
- // e.g. converting from client_request_state.end_time() (IMPALA-4440).
- lineage_graph.__set_ended(UnixMillis() / 1000);
-
- string lineage_record;
- LineageUtil::TLineageToJSON(lineage_graph, &lineage_record);
-
- if (AreQueryHooksEnabled()) {
- // invoke QueryEventHooks
- TQueryCompleteContext query_complete_context;
- query_complete_context.__set_lineage_string(lineage_record);
- const Status& status = exec_env_->frontend()->CallQueryCompleteHooks(
- query_complete_context);
-
- if (!status.ok()) {
- LOG(ERROR) << "Failed to send query lineage info to FE CallQueryCompleteHooks"
- << status.GetDetail();
- if (FLAGS_abort_on_failed_lineage_event) {
- CLEAN_EXIT_WITH_ERROR("Shutting down Impala Server due to "
- "abort_on_failed_lineage_event=true");
- }
- }
- }
-
- // lineage logfile writing is deprecated in favor of the
- // QueryEventHooks (see FE). this behavior is being retained
- // for now but may be removed in the future.
- if (IsLineageLoggingEnabled()) {
- const Status& status = lineage_logger_->AppendEntry(lineage_record);
- if (!status.ok()) {
- LOG(ERROR) << "Unable to record query lineage record: " << status.GetDetail();
- if (FLAGS_abort_on_failed_lineage_event) {
- CLEAN_EXIT_WITH_ERROR("Shutting down Impala Server due to "
- "abort_on_failed_lineage_event=true");
- }
- }
- return status;
- }
- return Status::OK();
-}
-
bool ImpalaServer::IsCoordinator() { return is_coordinator_; }
bool ImpalaServer::IsExecutor() { return is_executor_; }
@@ -639,81 +566,6 @@ Status ImpalaServer::InitLineageLogging() {
return Status::OK();
}
-Status ImpalaServer::LogAuditRecord(const ClientRequestState& request_state,
- const TExecRequest& request) {
- stringstream ss;
- rapidjson::StringBuffer buffer;
- rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
-
- writer.StartObject();
- // Each log entry is a timestamp mapped to a JSON object
- ss << UnixMillis();
- writer.String(ss.str().c_str());
- writer.StartObject();
- writer.String("query_id");
- writer.String(PrintId(request_state.query_id()).c_str());
- writer.String("session_id");
- writer.String(PrintId(request_state.session_id()).c_str());
- writer.String("start_time");
- writer.String(ToStringFromUnixMicros(request_state.start_time_us()).c_str());
- writer.String("authorization_failure");
- writer.Bool(Frontend::IsAuthorizationError(request_state.query_status()));
- writer.String("status");
- writer.String(request_state.query_status().GetDetail().c_str());
- writer.String("user");
- writer.String(request_state.effective_user().c_str());
- writer.String("impersonator");
- if (request_state.do_as_user().empty()) {
- // If there is no do_as_user() is empty, the "impersonator" field should be Null.
- writer.Null();
- } else {
- // Otherwise, the delegator is the current connected user.
- writer.String(request_state.connected_user().c_str());
- }
- writer.String("statement_type");
- if (request.stmt_type == TStmtType::DDL) {
- if (request.catalog_op_request.op_type == TCatalogOpType::DDL) {
- writer.String(
- PrintThriftEnum(request.catalog_op_request.ddl_params.ddl_type).c_str());
- } else {
- writer.String(PrintThriftEnum(request.catalog_op_request.op_type).c_str());
- }
- } else {
- writer.String(PrintThriftEnum(request.stmt_type).c_str());
- }
- writer.String("network_address");
- writer.String(TNetworkAddressToString(
- request_state.session()->network_address).c_str());
- writer.String("sql_statement");
- string stmt = replace_all_copy(request_state.sql_stmt(), "\n", " ");
- Redact(&stmt);
- writer.String(stmt.c_str());
- writer.String("catalog_objects");
- writer.StartArray();
- for (const TAccessEvent& event: request.access_events) {
- writer.StartObject();
- writer.String("name");
- writer.String(event.name.c_str());
- writer.String("object_type");
- writer.String(PrintThriftEnum(event.object_type).c_str());
- writer.String("privilege");
- writer.String(event.privilege.c_str());
- writer.EndObject();
- }
- writer.EndArray();
- writer.EndObject();
- writer.EndObject();
- Status status = audit_event_logger_->AppendEntry(buffer.GetString());
- if (!status.ok()) {
- LOG(ERROR) << "Unable to record audit event record: " << status.GetDetail();
- if (FLAGS_abort_on_failed_audit_event) {
- CLEAN_EXIT_WITH_ERROR("Shutting down Impala Server due to "
- "abort_on_failed_audit_event=true");
- }
- }
- return status;
-}
-
bool ImpalaServer::IsAuditEventLoggingEnabled() {
return !FLAGS_audit_event_log_dir.empty();
}
@@ -732,52 +584,6 @@ Status ImpalaServer::InitAuditEventLogging() {
return Status::OK();
}
-void ImpalaServer::LogQueryEvents(const ClientRequestState& request_state) {
- Status status = request_state.query_status();
- bool log_events = true;
- switch (request_state.stmt_type()) {
- case TStmtType::QUERY: {
- // If the query didn't finish, log audit and lineage events only if the
- // the client issued at least one fetch.
- if (!status.ok() && !request_state.fetched_rows()) log_events = false;
- break;
- }
- case TStmtType::DML: {
- if (!status.ok()) log_events = false;
- break;
- }
- case TStmtType::DDL: {
- if (request_state.catalog_op_type() == TCatalogOpType::DDL) {
- // For a DDL operation, log audit and lineage events only if the
- // operation finished.
- if (!status.ok()) log_events = false;
- } else {
- // This case covers local catalog operations such as SHOW and DESCRIBE.
- if (!status.ok() && !request_state.fetched_rows()) log_events = false;
- }
- break;
- }
- case TStmtType::EXPLAIN:
- case TStmtType::LOAD:
- case TStmtType::SET:
- case TStmtType::ADMIN_FN:
- default:
- break;
- }
- // Log audit events that are due to an AuthorizationException.
- if (IsAuditEventLoggingEnabled() &&
- (Frontend::IsAuthorizationError(request_state.query_status()) || log_events)) {
- // TODO: deal with an error status
- discard_result(LogAuditRecord(request_state, request_state.exec_request()));
- }
-
- if (log_events &&
- (AreQueryHooksEnabled() || IsLineageLoggingEnabled())) {
- // TODO: deal with an error status
- discard_result(LogLineageRecord(request_state));
- }
-}
-
Status ImpalaServer::InitProfileLogging() {
if (!FLAGS_log_query_to_file) return Status::OK();
@@ -796,6 +602,16 @@ Status ImpalaServer::InitProfileLogging() {
return Status::OK();
}
+Status ImpalaServer::AppendAuditEntry(const string& entry ) {
+ DCHECK(IsAuditEventLoggingEnabled());
+ return audit_event_logger_->AppendEntry(entry);
+}
+
+Status ImpalaServer::AppendLineageEntry(const string& entry ) {
+ DCHECK(IsLineageLoggingEnabled());
+ return lineage_logger_->AppendEntry(entry);
+}
+
Status ImpalaServer::GetRuntimeProfileOutput(const TUniqueId& query_id,
const string& user, TRuntimeProfileFormat::type format, stringstream* output,
TRuntimeProfileTree* thrift_output, Document* json_output) {
@@ -1339,9 +1155,6 @@ Status ImpalaServer::UnregisterQuery(const TUniqueId& query_id, bool check_infli
ImpaladMetrics::QUERY_DURATIONS->Update(duration_ms);
}
}
- // TODO (IMPALA-8572): move LogQueryEvents to before query unregistration
- LogQueryEvents(*request_state.get());
-
{
lock_guard<mutex> l(request_state->session()->lock);
request_state->session()->inflight_queries.erase(query_id);
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 684a18c..140c4a2 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -418,6 +418,9 @@ class ImpalaServer : public ImpalaServiceIf,
/// Returns true if query execution (FE) hooks are enabled, false otherwise.
bool AreQueryHooksEnabled();
+ /// Returns true if audit event logging is enabled, false otherwise.
+ bool IsAuditEventLoggingEnabled();
+
/// Retuns true if this is a coordinator, false otherwise.
bool IsCoordinator();
@@ -479,6 +482,12 @@ class ImpalaServer : public ImpalaServiceIf,
/// Convert the shutdown status to a human-readable string.
static std::string ShutdownStatusToString(const ShutdownStatusPB& shutdown_status);
+ /// Appends the audit_entry to audit_event_logger_.
+ Status AppendAuditEntry(const std::string& audit_entry);
+
+ /// Appends the lineage_entry to lineage_logger_.
+ Status AppendLineageEntry(const std::string& lineage_entry);
+
// Mapping between query option names and levels
QueryOptionLevels query_option_levels_;
@@ -765,18 +774,6 @@ class ImpalaServer : public ImpalaServiceIf,
/// exist. If there is any error creating the directory an error will be returned.
static Status InitLoggingDir(const std::string& log_dir) WARN_UNUSED_RESULT;
- /// Returns true if audit event logging is enabled, false otherwise.
- bool IsAuditEventLoggingEnabled();
-
- Status LogAuditRecord(
- const ClientRequestState& exec_state, const TExecRequest& request)
- WARN_UNUSED_RESULT;
-
- Status LogLineageRecord(const ClientRequestState& exec_state) WARN_UNUSED_RESULT;
-
- /// Log audit and column lineage events
- void LogQueryEvents(const ClientRequestState& exec_state);
-
/// Runs once every 5s to flush the profile log file to disk.
[[noreturn]] void LogFileFlushThread();
diff --git a/testdata/workloads/functional-query/queries/QueryTest/lineage.test b/testdata/workloads/functional-query/queries/QueryTest/lineage.test
new file mode 100644
index 0000000..2335d5a
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/lineage.test
@@ -0,0 +1,5648 @@
+====
+---- QUERY
+drop database if exists lineage_test_db cascade
+====
+---- QUERY
+create database lineage_test_db
+====
+---- QUERY
+create table lineage_test_db.alltypes like functional_hbase.alltypes
+====
+---- QUERY
+create table lineage_test_db.alltypesnopart like functional.alltypesnopart
+====
+---- QUERY
+create table lineage_test_db.alltypessmall like functional.alltypessmall
+====
+---- QUERY
+create table lineage_test_db.alltypesinsert like functional.alltypesinsert
+====
+---- QUERY
+create view lineage_test_db.alltypes_view as select * from lineage_test_db.alltypes
+====
+---- LINEAGE
+{
+ "hash": "aba4b39bf3c34b5a9ce8fd782acc0a26",
+ "timestamp": 1568097481,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "x",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1,
+ 2,
+ 3
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "eb4368837f0c4468:ef6841f500000000",
+ "user": "bharath",
+ "queryText": "select * from ( select tinyint_col + int_col x from functional.alltypes union all select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2",
+ "endTime": 1568097481
+}
+---- QUERY
+# Simple query to introduce projection dependencies. Shows resolution
+# through inline views, unions, and materialization points.
+select * from (
+ select tinyint_col + int_col x from functional.alltypes
+ union all
+ select sum(bigint_col) y from (select bigint_col from functional.alltypes) v1) v2
+====
+---- LINEAGE
+{
+ "hash": "2540c828f7397099c11558544485e97c",
+ "timestamp": 1568097487,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "sum(a.tinyint_col) OVER(...)",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.smallint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "count(b.string_col)",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.string_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 6
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.timestamp_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.year",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.bigint_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.float_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.id",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 2,
+ 3
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 2,
+ 3,
+ 5,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12
+ ],
+ "targets": [
+ 0,
+ 4,
+ 6
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "b04c96bc7cd037bf:950c0e9700000000",
+ "user": "bharath",
+ "queryText": "select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), count(b.string_col), b.timestamp_col from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) where a.year = 2010 and b.float_col > 0 group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col having count(a.int_col) > 10 order by b.bigint_col limit 10",
+ "endTime": 1568097487
+}
+---- QUERY
+# Simple query to introduce predicate dependencies. Shows conjuncts in WHERE, ON, and
+# HAVING clause, as well dependencies introduced by GROUP BY and ORDER BY clauses and
+# analytic functions.
+select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),
+ count(b.string_col), b.timestamp_col
+from functional.alltypes a join functional.alltypessmall b on (a.id = b.id)
+where a.year = 2010 and b.float_col > 0
+group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col
+having count(a.int_col) > 10
+order by b.bigint_col limit 10
+====
+---- LINEAGE
+{
+ "hash": "b9d4145e873d7ee55e485690a6465421",
+ "timestamp": 1568097492,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097492,
+ "tableName": "lineage_test_db.lineage_test_tbl1"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097492,
+ "tableName": "lineage_test_db.lineage_test_tbl1"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "cd4196096bc21335:cc080f1c00000000",
+ "user": "bharath",
+ "queryText": "create table lineage_test_db.lineage_test_tbl1 as select int_col, tinyint_col from functional.alltypes",
+ "endTime": 1568097493
+}
+---- QUERY
+# CTAS queries
+create table lineage_test_db.lineage_test_tbl1 as select int_col, tinyint_col from functional.alltypes
+====
+---- LINEAGE
+{
+ "hash": "bbd2400879f7304bba4182220c138651",
+ "timestamp": 1568097498,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097498,
+ "tableName": "lineage_test_db.lineage_test_tbl2"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097498,
+ "tableName": "lineage_test_db.lineage_test_tbl2"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.string_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.year",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.id",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.month",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 4,
+ 5,
+ 6,
+ 7
+ ],
+ "targets": [
+ 0,
+ 2
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "69419851b45132ca:21275fb100000000",
+ "user": "bharath",
+ "queryText": "create table lineage_test_db.lineage_test_tbl2 as select distinct a.int_col, a.string_col from functional.alltypes a inner join functional.alltypessmall b on (a.id = b.id) where a.year = 2009 and b.month = 2",
+ "endTime": 1568097499
+}
+---- QUERY
+create table lineage_test_db.lineage_test_tbl2 as
+select distinct a.int_col, a.string_col from functional.alltypes a
+inner join functional.alltypessmall b on (a.id = b.id)
+where a.year = 2009 and b.month = 2
+====
+---- LINEAGE
+{
+ "hash": "8f43e2adb3da2eac5d00d1eba96bb553",
+ "timestamp": 1568097504,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097504,
+ "tableName": "lineage_test_db.lineage_test_tbl3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "bc4fbd39b756403c:101d778c00000000",
+ "user": "bharath",
+ "queryText": "create table lineage_test_db.lineage_test_tbl3 as select * from (select * from (select int_col from functional.alltypestiny limit 1) v1 ) v2",
+ "endTime": 1568097504
+}
+---- QUERY
+create table lineage_test_db.lineage_test_tbl3 as
+select * from
+ (select * from
+ (select int_col from functional.alltypestiny limit 1) v1 ) v2
+====
+---- LINEAGE
+{
+ "hash": "502d2de3774289b12a705777da61baae",
+ "timestamp": 1568097509,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.bigint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.bool_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.date_string_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.double_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.float_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.int_col",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.month",
+ "id": 15,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 16,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.smallint_col",
+ "id": 17,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 18,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.string_col",
+ "id": 19,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 20,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.timestamp_col",
+ "id": 21,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 22,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.tinyint_col",
+ "id": 23,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 24,
+ "metadata": {
+ "tableCreateTime": 1568097510,
+ "tableName": "lineage_test_db.lineage_test_tbl4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypes.year",
+ "id": 25,
+ "metadata": {
+ "tableCreateTime": 1568094417,
+ "tableName": "functional_hbase.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 9
+ ],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 11
+ ],
+ "targets": [
+ 10
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 13
+ ],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 15
+ ],
+ "targets": [
+ 14
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 17
+ ],
+ "targets": [
+ 16
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 19
+ ],
+ "targets": [
+ 18
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 21
+ ],
+ "targets": [
+ 20
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 23
+ ],
+ "targets": [
+ 22
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 25
+ ],
+ "targets": [
+ 24
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "774dfa27d60b670b:37bf45ec00000000",
+ "user": "bharath",
+ "queryText": "create table lineage_test_db.lineage_test_tbl4 as select * from functional_hbase.alltypes limit 5",
+ "endTime": 1568097511
+}
+---- QUERY
+# CTAS from HBase table
+create table lineage_test_db.lineage_test_tbl4 as select * from functional_hbase.alltypes limit 5
+====
+---- LINEAGE
+{
+ "hash": "2fa5d0ab4e1c38a9d7abde595f4c60c8",
+ "timestamp": 1568097516,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568097478,
+ "tableName": "lineage_test_db.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 1
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 3
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 5
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 10
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "454b0e7178b98fe4:6d85e21300000000",
+ "user": "bharath",
+ "queryText": "insert into lineage_test_db.alltypes values (1, 1, true, \"1999-12-01\", 2.0, 1.0, 1, 12, 2, \"abs\", cast(now() as timestamp), 1, 1999)",
+ "endTime": 1568097516
+}
+---- QUERY
+# Insert into an HBase table
+insert into
+lineage_test_db.alltypes
+ values (1, 1, true, "1999-12-01", 2.0, 1.0, 1, 12, 2, "abs",
+ cast(now() as timestamp), 1, 1999)
+====
+---- LINEAGE
+{
+ "hash": "e3af61c7dc3e8cd9ab632c3c935df032",
+ "timestamp": 1568097521,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bool_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568097479,
+ "tableName": "lineage_test_db.alltypesnopart"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.timestamp_col",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 5
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 10
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 13
+ ],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "ad4f9e564b46993a:df41be2500000000",
+ "user": "bharath",
+ "queryText": "insert into table lineage_test_db.alltypesnopart (id, bool_col, timestamp_col) select id, bool_col, timestamp_col from functional.alltypes",
+ "endTime": 1568097521
+}
+---- QUERY
+insert into table lineage_test_db.alltypesnopart (id, bool_col, timestamp_col)
+select id, bool_col, timestamp_col
+from functional.alltypes
+====
+---- LINEAGE
+{
+ "hash": "453abb733b2b21ba827f80897fe3b579",
+ "timestamp": 1568097527,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.smallint_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.month",
+ "id": 15,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.year",
+ "id": 16,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 1
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 3
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 5
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 8
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 10
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 13
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 14
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 15,
+ 16
+ ],
+ "targets": [
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 7,
+ 9,
+ 10,
+ 11,
+ 12,
+ 13,
+ 14
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "b04704db216d74a2:b2a1d75a00000000",
+ "user": "bharath",
+ "queryText": "insert into table lineage_test_db.alltypessmall (smallint_col, int_col) partition (year=2009, month=04) select smallint_col, int_col from functional.alltypes where year=2009 and month=05",
+ "endTime": 1568097527
+}
+---- QUERY
+insert into table lineage_test_db.alltypessmall (smallint_col, int_col)
+partition (year=2009, month=04)
+select smallint_col, int_col
+from functional.alltypes
+where year=2009 and month=05
+====
+---- LINEAGE
+{
+ "hash": "b6bcec64b3eace721772f4b1d4e2cb57",
+ "timestamp": 1568097532,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_seq_snap.alltypes.year",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093699,
+ "tableName": "functional_seq_snap.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_seq_snap.alltypes.month",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093699,
+ "tableName": "functional_seq_snap.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_seq_snap.alltypes.id",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093699,
+ "tableName": "functional_seq_snap.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_seq_snap.alltypes.int_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093699,
+ "tableName": "functional_seq_snap.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 15,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_seq_snap.alltypes.string_col",
+ "id": 16,
+ "metadata": {
+ "tableCreateTime": 1568093699,
+ "tableName": "functional_seq_snap.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 17,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 10
+ ],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 13
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 14
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 16
+ ],
+ "targets": [
+ 15
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 17
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4,
+ 6,
+ 7,
+ 8,
+ 9,
+ 11,
+ 12,
+ 13,
+ 14,
+ 15,
+ 17
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "0848da428c8561da:15695de800000000",
+ "user": "bharath",
+ "queryText": "insert into table lineage_test_db.alltypessmall (id, string_col, int_col) partition (year, month) select id, string_col, int_col, year, month from functional_seq_snap.alltypes where year=2009 and month>10",
+ "endTime": 1568097533
+}
+---- QUERY
+insert into table lineage_test_db.alltypessmall (id, string_col, int_col)
+partition (year, month)
+select id, string_col, int_col, year, month
+from functional_seq_snap.alltypes
+where year=2009 and month>10
+====
+---- LINEAGE
+{
+ "hash": "2b82f76becf76311633c26498d9f57d8",
+ "timestamp": 1568097538,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.month",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bool_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.smallint_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 15,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.float_col",
+ "id": 16,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 17,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.double_col",
+ "id": 18,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 19,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.date_string_col",
+ "id": 20,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 21,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.string_col",
+ "id": 22,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 23,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.timestamp_col",
+ "id": 24,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.year",
+ "id": 25,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 2
+ ],
+ "targets": [
+ 1
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 4
+ ],
+ "targets": [
+ 3
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 5
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 8
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 10
+ ],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 12
+ ],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 14
+ ],
+ "targets": [
+ 13
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 16
+ ],
+ "targets": [
+ 15
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 18
+ ],
+ "targets": [
+ 17
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 20
+ ],
+ "targets": [
+ 19
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 22
+ ],
+ "targets": [
+ 21
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 24
+ ],
+ "targets": [
+ 23
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 2,
+ 4,
+ 25
+ ],
+ "targets": [
+ 0,
+ 1,
+ 3,
+ 5,
+ 7,
+ 9,
+ 11,
+ 13,
+ 15,
+ 17,
+ 19,
+ 21,
+ 23
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "c64f079557552c0b:13fdd21300000000",
+ "user": "bharath",
+ "queryText": "insert into table lineage_test_db.alltypessmall partition (year=2009, month) select min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col), min(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col), min(timestamp_col), month from functional.alltypes where year=2009 and month>10 group by month having min(id) > 10",
+ "endTime": 1568097538
+}
+---- QUERY
+insert into table lineage_test_db.alltypessmall
+partition (year=2009, month)
+select min(id), min(bool_col), min(tinyint_col), min(smallint_col), min(int_col),
+min(bigint_col), min(float_col), min(double_col), min(date_string_col), min(string_col),
+min(timestamp_col), month
+from functional.alltypes
+where year=2009 and month>10
+group by month
+having min(id) > 10
+====
+---- LINEAGE
+{
+ "hash": "33162d783545a2bdc1e8a3a95553971f",
+ "timestamp": 1568097543,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "max(tinyint_col) OVER(...)",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 2
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "71437ef9b661a370:78d5443c00000000",
+ "user": "bharath",
+ "queryText": "select max(tinyint_col) over(partition by int_col) from functional.alltypes group by int_col, tinyint_col",
+ "endTime": 1568097544
+}
+---- QUERY
+# Select statements with analytic functions
+select
+max(tinyint_col) over(partition by int_col)
+from functional.alltypes
+group by int_col, tinyint_col
+====
+---- LINEAGE
+{
+ "hash": "4f1ecaaed571d2ed9f09f091f399c311",
+ "timestamp": 1568097549,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "rank() OVER(...)",
+ "id": 2
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "af40268c951b5907:486c9bf100000000",
+ "user": "bharath",
+ "queryText": "select int_col, rank() over(order by int_col) from functional.alltypesagg",
+ "endTime": 1568097549
+}
+---- QUERY
+select int_col, rank() over(order by int_col) from functional.alltypesagg
+====
+---- LINEAGE
+{
+ "hash": "83067b3ed65f566ee2a7c97259b3f485",
+ "timestamp": 1568097554,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "count(a.double_col) OVER(...)",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.double_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.id",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 6,
+ 7
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "254e0097e54345b3:17adb7a400000000",
+ "user": "bharath",
+ "queryText": "select a.tinyint_col, a.int_col, count(a.double_col) over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following) from functional.alltypes a inner join functional.alltypessmall b on a.id = b.id order by a.tinyint_col, a.int_col",
+ "endTime": 1568097555
+}
+---- QUERY
+select a.tinyint_col, a.int_col, count(a.double_col)
+ over(partition by a.tinyint_col order by a.int_col desc rows between 1 preceding and 1 following)
+from functional.alltypes a inner join functional.alltypessmall b on a.id = b.id
+order by a.tinyint_col, a.int_col
+====
+---- LINEAGE
+{
+ "hash": "7bda479e44e923018a8e83435a4e7567",
+ "timestamp": 1568097560,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.double_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "b",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bool_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a + b",
+ "id": 7
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col + a",
+ "id": 8
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3,
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 3,
+ 5
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "bb4227a0b4b7f483:759587c600000000",
+ "user": "bharath",
+ "queryText": "with v2 as (select double_col, count(int_col) over() a, sum(int_col + bigint_col) over(partition by bool_col) b from (select * from functional.alltypes) v1) select double_col, a, b, a + b, double_col + a from v2 order by 2, 3, 4",
+ "endTime": 1568097560
+}
+---- QUERY
+with v2 as
+ (select
+ double_col,
+ count(int_col) over() a,
+ sum(int_col + bigint_col) over(partition by bool_col) b
+ from
+ (select * from functional.alltypes) v1)
+select double_col, a, b, a + b, double_col + a from v2
+order by 2, 3, 4
+====
+---- LINEAGE
+{
+ "hash": "f91d8724a1537af46652415e14f0067e",
+ "timestamp": 1568097565,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.double_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "b",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bool_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a + b",
+ "id": 7
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col + a",
+ "id": 8
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3,
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 3,
+ 5
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "e349f417ef593b44:f75a453900000000",
+ "user": "bharath",
+ "queryText": "select double_col, a, b, a + b, double_col + a from (select double_col, count(int_col) over() a, sum(int_col + bigint_col) over(partition by bool_col) b from (select * from functional.alltypes) v1) v2 order by 2, 3, 4",
+ "endTime": 1568097565
+}
+---- QUERY
+select double_col, a, b, a + b, double_col + a from
+ (select
+ double_col,
+ count(int_col) over() a,
+ sum(int_col + bigint_col) over(partition by bool_col) b
+ from
+ (select * from functional.alltypes) v1) v2
+order by 2, 3, 4
+====
+---- LINEAGE
+{
+ "hash": "43098e753e37a70f008726178e10dadf",
+ "timestamp": 1568097570,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.month",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.year",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.int_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.month",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 6
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "054c112eb283a265:317a9e3700000000",
+ "user": "bharath",
+ "queryText": "select a.month, a.year, b.int_col, b.month from (select year, month from functional.alltypes union all select year, month from functional.alltypes) a inner join functional.alltypessmall b on (a.month = b.month) where b.month = 1",
+ "endTime": 1568097571
+}
+---- QUERY
+# Union statements
+select a.month, a.year, b.int_col, b.month
+from
+ (select year, month from functional.alltypes
+ union all
+ select year, month from functional.alltypes) a
+ inner join
+ functional.alltypessmall b
+ on (a.month = b.month)
+where b.month = 1
+====
+---- LINEAGE
+{
+ "hash": "20cdd5a4921f1d865bcc1f8b984839ae",
+ "timestamp": 1568097576,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.month",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "t2.int_col + 1",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.month",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3,
+ 5,
+ 6
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "724e262b77839d11:a27f72be00000000",
+ "user": "bharath",
+ "queryText": "select t1.int_col, t2.month, t2.int_col + 1 from ( select int_col, count(*) from functional.alltypessmall where month = 1 group by int_col having count(*) > 1 order by count(*) desc limit 5 ) t1 join functional.alltypes t2 on (t1.int_col = t2.int_col) where month = 1",
+ "endTime": 1568097576
+}
+---- QUERY
+# Select statements with inline views
+select t1.int_col, t2.month, t2.int_col + 1
+from (
+ select int_col, count(*)
+ from functional.alltypessmall
+ where month = 1
+ group by int_col
+ having count(*) > 1
+ order by count(*) desc limit 5
+ ) t1
+join functional.alltypes t2 on (t1.int_col = t2.int_col)
+where month = 1
+====
+---- LINEAGE
+{
+ "hash": "1b739655724090be8f3dd8bb018fe536",
+ "timestamp": 1568097581,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.smallint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.id",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.tinyint_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 6
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.int_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 8
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.float_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 10
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.string_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.day",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.month",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 9
+ ],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 11
+ ],
+ "targets": [
+ 10
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3,
+ 5,
+ 7,
+ 9,
+ 11,
+ 12,
+ 13
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4,
+ 6,
+ 8,
+ 10
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "394bc5f6a9764ca2:9cd14c5f00000000",
+ "user": "bharath",
+ "queryText": "select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col from functional.alltypessmall c join ( select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, a.int_col int_col, a.month month, b.float_col float_col, b.id id from ( select * from functional.alltypesagg a where month=1 ) a join functional.alltypessmall b on (a.smallint_col = b.id) ) x on (x.tinyint_col = c.id) where x.day=1 and x.int_col > 899 and [...]
+ "endTime": 1568097582
+}
+---- QUERY
+select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col
+from functional.alltypessmall c
+join (
+ select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day,
+ a.int_col int_col, a.month month, b.float_col float_col, b.id id
+ from ( select * from functional.alltypesagg a where month=1 ) a
+ join functional.alltypessmall b on (a.smallint_col = b.id)
+ ) x on (x.tinyint_col = c.id)
+where x.day=1
+and x.int_col > 899
+and x.float_col > 4.5
+and c.string_col < '7'
+and x.int_col + x.float_col + cast(c.string_col as float) < 1000
+====
+---- LINEAGE
+{
+ "hash": "b91dd6944864a3c14ecefb697ca95459",
+ "timestamp": 1568097587,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "c1",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypessmall.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568094418,
+ "tableName": "functional_hbase.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "c2",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypessmall.float_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568094418,
+ "tableName": "functional_hbase.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "c3",
+ "id": 4
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "884c215284a67dd0:d3e3af0e00000000",
+ "user": "bharath",
+ "queryText": "select c1, c2, c3 from (select c1, c2, c3 from (select int_col c1, sum(float_col) c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1) x order by 2,3 desc limit 5 ) y",
+ "endTime": 1568097587
+}
+---- QUERY
+select c1, c2, c3
+from
+ (select c1, c2, c3
+ from
+ (select int_col c1, sum(float_col) c2, min(float_col) c3
+ from functional_hbase.alltypessmall
+ group by 1) x
+ order by 2,3 desc
+ limit 5
+) y
+====
+---- LINEAGE
+{
+ "hash": "e59cf0e0de9015036a15df76e9e61c85",
+ "timestamp": 1568097593,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "c1",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypessmall.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568094418,
+ "tableName": "functional_hbase.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "x2",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypessmall.tinyint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568094418,
+ "tableName": "functional_hbase.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 0,
+ 2
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "a240cc4e302873cd:798d478b00000000",
+ "user": "bharath",
+ "queryText": "select c1, x2 from ( select c1, min(c2) x2 from ( select c1, c2, c3 from ( select int_col c1, tinyint_col c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1, 2 order by 1,2 limit 1 ) x ) x2 group by c1 ) y order by 2,1 desc limit 0",
+ "endTime": 1568097593
+}
+---- QUERY
+select c1, x2
+from (
+ select c1, min(c2) x2
+ from (
+ select c1, c2, c3
+ from (
+ select int_col c1, tinyint_col c2, min(float_col) c3
+ from functional_hbase.alltypessmall
+ group by 1, 2
+ order by 1,2
+ limit 1
+ ) x
+ ) x2
+ group by c1
+) y
+order by 2,1 desc
+limit 0
+====
+---- LINEAGE
+{
+ "hash": "cf15009df42b7aea0f6abaeea3381b85",
+ "timestamp": 1568097598,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "c74a33f33cef2014:feaa957900000000",
+ "user": "bharath",
+ "queryText": "select id from functional.view_view",
+ "endTime": 1568097598
+}
+---- QUERY
+# Views
+select id from functional.view_view
+====
+---- LINEAGE
+{
+ "hash": "5dc06563b62b39278652242034a2a70d",
+ "timestamp": 1568097603,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "5f46e701ed4868bc:b58d892a00000000",
+ "user": "bharath",
+ "queryText": "select t.id from (select id from functional.alltypes_view) t where t.id < 10",
+ "endTime": 1568097603
+}
+---- QUERY
+select t.id from (select id from functional.alltypes_view) t
+where t.id < 10
+====
+---- LINEAGE
+{
+ "hash": "c9e45730cafb6910b54eaa5c53dc44b1",
+ "timestamp": 1568097608,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.string_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.float_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bool_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.id",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 6,
+ 7
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "8945d3c9252417af:74f47ae100000000",
+ "user": "bharath",
+ "queryText": "select string_col, float_col, bool_col from functional.alltypes where id in (select id from functional.alltypesagg)",
+ "endTime": 1568097609
+}
+---- QUERY
+# Subqueries
+select string_col, float_col, bool_col
+from functional.alltypes
+where id in
+ (select id from functional.alltypesagg)
+====
+---- LINEAGE
+{
+ "hash": "1d2fa8025e9279488af58b5b8dd614d9",
+ "timestamp": 1568097614,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "1",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.tinyint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.tinyint_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 2
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "004853208b5ec985:38f0fe7a00000000",
+ "user": "bharath",
+ "queryText": "select 1 from functional.alltypesagg a where exists (select id, count(int_col) over (partition by bool_col) from functional.alltypestiny b where a.tinyint_col = b.tinyint_col group by id, int_col, bool_col) and tinyint_col < 10",
+ "endTime": 1568097614
+}
+---- QUERY
+select 1
+from functional.alltypesagg a
+where exists
+ (select id, count(int_col) over (partition by bool_col)
+ from functional.alltypestiny b
+ where a.tinyint_col = b.tinyint_col
+ group by id, int_col, bool_col)
+and tinyint_col < 10
+====
+---- LINEAGE
+{
+ "hash": "7e678cde53c9d120a6f79134551a8c41",
+ "timestamp": 1568097619,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col + 1",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col - 1",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.bool_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.int_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 4,
+ 5,
+ 6
+ ],
+ "targets": [
+ 0,
+ 2
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "5c45bae6daa5b536:110be36c00000000",
+ "user": "bharath",
+ "queryText": "select int_col + 1, tinyint_col - 1 from functional.alltypes a where a.int_col < (select max(int_col) from functional.alltypesagg g where g.bool_col = true) and a.bigint_col > 10",
+ "endTime": 1568097619
+}
+---- QUERY
+select int_col + 1, tinyint_col - 1
+from functional.alltypes a
+where a.int_col <
+ (select max(int_col) from functional.alltypesagg g where g.bool_col = true)
+and a.bigint_col > 10
+====
+---- LINEAGE
+{
+ "hash": "a7ab58d90540f28a8dfd69703632ad7a",
+ "timestamp": 1568097624,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "x",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "y",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "214d40171885157a:b422c0b100000000",
+ "user": "bharath",
+ "queryText": "with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t",
+ "endTime": 1568097624
+}
+---- QUERY
+# With clause
+with t as (select int_col x, bigint_col y from functional.alltypes) select x, y from t
+====
+---- LINEAGE
+{
+ "hash": "59c56529d835660a4f776c0edc60ecc1",
+ "timestamp": 1568097629,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.year",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.month",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.id",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bool_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "tinyint_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "smallint_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "int_col",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.int_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "bigint_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "float_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "double_col",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "date_string_col",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "string_col",
+ "id": 15,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.string_col",
+ "id": 16,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 17,
+ "metadata": {
+ "tableCreateTime": 1568097480,
+ "tableName": "lineage_test_db.alltypesinsert"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 10
+ ],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 13
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 14
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 16
+ ],
+ "targets": [
+ 15
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [],
+ "targets": [
+ 17
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "1f4b1f66d4cc207e:f0435aee00000000",
+ "user": "bharath",
+ "queryText": "with t1 as (select * from functional.alltypestiny) insert into lineage_test_db.alltypesinsert (id, int_col, string_col) partition(year, month) select id, int_col, string_col, year, month from t1",
+ "endTime": 1568097630
+}
+---- QUERY
+with t1 as (select * from functional.alltypestiny)
+insert into lineage_test_db.alltypesinsert (id, int_col, string_col) partition(year, month)
+select id, int_col, string_col, year, month from t1
+====
+---- LINEAGE
+{
+ "hash": "36f0326d13fc4d2bd03196549624a15a",
+ "timestamp": 1568097635,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "lead(a, 1, NULL) OVER(...)",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bigint_col",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.bool_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.string_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "2a4397fae87ec604:390fe7d600000000",
+ "user": "bharath",
+ "queryText": "select lead(a) over (partition by b order by c) from (select lead(id) over (partition by int_col order by bigint_col) as a, max(id) over (partition by tinyint_col order by int_col) as b, min(int_col) over (partition by string_col order by bool_col) as c from functional.alltypes) v",
+ "endTime": 1568097636
+}
+---- QUERY
+# Nested analytic functions
+select lead(a) over (partition by b order by c)
+from
+ (select lead(id) over (partition by int_col order by bigint_col) as a,
+ max(id) over (partition by tinyint_col order by int_col) as b,
+ min(int_col) over (partition by string_col order by bool_col) as c
+ from functional.alltypes) v
+====
+---- LINEAGE
+{
+ "hash": "f6683b020d20b90902603bbe7f6635e8",
+ "timestamp": 1568097641,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097641,
+ "tableName": "lineage_test_db.lineage_test_view1"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "a14c35049f6dde68:643393e500000000",
+ "user": "bharath",
+ "queryText": "create view lineage_test_db.lineage_test_view1 as select id from functional.alltypestiny",
+ "endTime": 1568097641
+}
+---- QUERY
+# Create view statement (IMPALA-4219)
+create view lineage_test_db.lineage_test_view1 as select id from functional.alltypestiny
+====
+---- LINEAGE
+{
+ "hash": "d1217e5cde0f930466e6ccfe7c261a53",
+ "timestamp": 1568097646,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097646,
+ "tableName": "lineage_test_db.lineage_test_view2"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypessmall.int_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568094418,
+ "tableName": "functional_hbase.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "b",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097646,
+ "tableName": "lineage_test_db.lineage_test_view2"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional_hbase.alltypessmall.tinyint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568094418,
+ "tableName": "functional_hbase.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3
+ ],
+ "targets": [
+ 0,
+ 2
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "774c6a2b13d69002:ee0b793e00000000",
+ "user": "bharath",
+ "queryText": "create view lineage_test_db.lineage_test_view2 (a, b) as select c1, x2 from ( select c1, min(c2) x2 from ( select c1, c2, c3 from ( select int_col c1, tinyint_col c2, min(float_col) c3 from functional_hbase.alltypessmall group by 1, 2 order by 1,2 limit 1 ) x ) x2 group by c1 ) y order by 2,1 desc limit 0",
+ "endTime": 1568097646
+}
+---- QUERY
+# Create view with nested inline views (IMPALA-4219)
+create view lineage_test_db.lineage_test_view2 (a, b) as select c1, x2
+from (
+ select c1, min(c2) x2
+ from (
+ select c1, c2, c3
+ from (
+ select int_col c1, tinyint_col c2, min(float_col) c3
+ from functional_hbase.alltypessmall
+ group by 1, 2
+ order by 1,2
+ limit 1
+ ) x
+ ) x2
+ group by c1
+) y
+order by 2,1 desc
+limit 0
+====
+---- LINEAGE
+{
+ "hash": "7fbb84397e7103b32ee785085027ffcf",
+ "timestamp": 1568097651,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a1",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.smallint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a2",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.id",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a3",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.tinyint_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a4",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a5",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.int_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a6",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.float_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "a7",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568097651,
+ "tableName": "lineage_test_db.lineage_test_view3"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.string_col",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.day",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypesagg.month",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568093721,
+ "tableName": "functional.alltypesagg"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 8
+ ],
+ "targets": [
+ 7
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 10
+ ],
+ "targets": [
+ 9
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 12
+ ],
+ "targets": [
+ 11
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 3,
+ 5,
+ 8,
+ 10,
+ 12,
+ 13,
+ 14
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4,
+ 6,
+ 7,
+ 9,
+ 11
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "bb458a67da0ec51a:8bacb37800000000",
+ "user": "bharath",
+ "queryText": "create view lineage_test_db.lineage_test_view3 (a1, a2, a3, a4, a5, a6, a7) as select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col from functional.alltypessmall c join ( select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day, a.int_col int_col, a.month month, b.float_col float_col, b.id id from ( select * from functional.alltypesagg a where month=1 ) a join functional.alltypessmall b on ( [...]
+ "endTime": 1568097652
+}
+---- QUERY
+# Create view with complex select statement (IMPALA-4219)
+create view lineage_test_db.lineage_test_view3 (a1, a2, a3, a4, a5, a6, a7) as
+ select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col
+ from functional.alltypessmall c
+ join (
+ select a.smallint_col smallint_col, a.tinyint_col tinyint_col, a.day day,
+ a.int_col int_col, a.month month, b.float_col float_col, b.id id
+ from ( select * from functional.alltypesagg a where month=1 ) a
+ join functional.alltypessmall b on (a.smallint_col = b.id)
+ ) x on (x.tinyint_col = c.id)
+ where x.day=1
+ and x.int_col > 899
+ and x.float_col > 4.5
+ and c.string_col < '7'
+ and x.int_col + x.float_col + cast(c.string_col as float) < 1000
+====
+---- LINEAGE
+{
+ "hash": "80ed48e4969ac7e03d08d5eb6600f4bf",
+ "timestamp": 1568097657,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "_c0",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097657,
+ "tableName": "lineage_test_db.lineage_test_view4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.tinyint_col",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.smallint_col",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "_c1",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568097657,
+ "tableName": "lineage_test_db.lineage_test_view4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.string_col",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "timestamp_col",
+ "id": 6,
+ "metadata": {
+ "tableCreateTime": 1568097657,
+ "tableName": "lineage_test_db.lineage_test_view4"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.timestamp_col",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.int_col",
+ "id": 8,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.year",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.bigint_col",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.float_col",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypessmall.id",
+ "id": 12,
+ "metadata": {
+ "tableCreateTime": 1568093682,
+ "tableName": "functional.alltypessmall"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 2,
+ 3
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 2,
+ 3,
+ 5,
+ 7,
+ 8,
+ 9,
+ 10,
+ 11,
+ 12
+ ],
+ "targets": [
+ 0,
+ 4,
+ 6
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "55492fe4dc145adc:7c69c17b00000000",
+ "user": "bharath",
+ "queryText": "create view lineage_test_db.lineage_test_view4 as select * from ( select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id), count(b.string_col), b.timestamp_col from functional.alltypes a join functional.alltypessmall b on (a.id = b.id) where a.year = 2010 and b.float_col > 0 group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col having count(a.int_col) > 10 order by b.bigint_col limit 10) t",
+ "endTime": 1568097657
+}
+---- QUERY
+# Create view with select statement containing an analytic function (IMPALA-4219)
+create view lineage_test_db.lineage_test_view4 as
+ select * from (
+ select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),
+ count(b.string_col), b.timestamp_col
+ from functional.alltypes a join functional.alltypessmall b on (a.id = b.id)
+ where a.year = 2010 and b.float_col > 0
+ group by a.tinyint_col, a.smallint_col, a.id, b.string_col, b.timestamp_col, b.bigint_col
+ having count(a.int_col) > 10
+ order by b.bigint_col limit 10) t
+====
+---- LINEAGE
+{
+ "hash": "561916b21b73cf61397e1ca9aa2fb682",
+ "timestamp": 1568097662,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097481,
+ "tableName": "lineage_test_db.alltypes_view"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypestiny.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093689,
+ "tableName": "functional.alltypestiny"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ }
+ ],
+ "queryId": "21456fbe86072029:b489dd3400000000",
+ "user": "bharath",
+ "queryText": "alter view lineage_test_db.alltypes_view as select id from functional.alltypestiny",
+ "endTime": 1568097663
+}
+---- QUERY
+# Alter view statement (IMPALA-4219)
+alter view lineage_test_db.alltypes_view as select id from functional.alltypestiny
+====
+---- LINEAGE
+{
+ "hash": "e771fd2a8795e5aaad3e86c266b9a039",
+ "timestamp": 1568097668,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "x",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.complex_struct_col.f1",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_struct_col.f1",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_struct_col.f2",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.nested_struct_col.f2.f12.f21",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.year",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1,
+ 2,
+ 3
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 4,
+ 5
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "09445f3d714f391c:18975f3400000000",
+ "user": "bharath",
+ "queryText": "select * from ( select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes where year = 2000 order by nested_struct_col.f2.f12.f21 limit 10 union all select sum(f1) y from (select complex_struct_col.f1 f1 from functional.allcomplextypes group by 1) v1) v2",
+ "endTime": 1568097668
+}
+---- QUERY
+# Test references to struct fields.
+select * from (
+ select int_struct_col.f1 + int_struct_col.f2 x from functional.allcomplextypes
+ where year = 2000
+ order by nested_struct_col.f2.f12.f21 limit 10
+ union all
+ select sum(f1) y from
+ (select complex_struct_col.f1 f1 from functional.allcomplextypes
+ group by 1) v1) v2
+====
+---- LINEAGE
+{
+ "hash": "63df7e3a395b2f9d56928ccd655edef5",
+ "timestamp": 1568097673,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "item",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_array_col.item",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "key",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.key",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "f1",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.value.f1",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "f2",
+ "id": 6
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.value.f2",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 5
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4,
+ 6
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "0344bc3c25ad0f7f:2f22a35700000000",
+ "user": "bharath",
+ "queryText": "select * from functional.allcomplextypes.int_array_col a inner join functional.allcomplextypes.struct_map_col m on (a.item = m.f1)",
+ "endTime": 1568097673
+}
+---- QUERY
+# Test absolute collection table refs. Ensure that the vertex ids of sources are
+# always an absolute explicit path.
+select * from functional.allcomplextypes.int_array_col a inner join
+ functional.allcomplextypes.struct_map_col m on (a.item = m.f1)
+====
+---- LINEAGE
+{
+ "hash": "6df49e42edeccad1139432ad33ad037a",
+ "timestamp": 1568097678,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "year",
+ "id": 2
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.year",
+ "id": 3,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "month",
+ "id": 4
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.month",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "item",
+ "id": 6
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_array_col.item",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "key",
+ "id": 8
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.key",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "f1",
+ "id": 10
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.value.f1",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "f2",
+ "id": 12
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.value.f2",
+ "id": 13,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_array_col",
+ "id": 14,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col",
+ "id": 15,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 3
+ ],
+ "targets": [
+ 2
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 4
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 9
+ ],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 11
+ ],
+ "targets": [
+ 10
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 13
+ ],
+ "targets": [
+ 12
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 7,
+ 11,
+ 14,
+ 15
+ ],
+ "targets": [
+ 0,
+ 2,
+ 4,
+ 6,
+ 8,
+ 10,
+ 12
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "c942b32cc2a9e45e:47fc681100000000",
+ "user": "bharath",
+ "queryText": "select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m where a.item = m.f1",
+ "endTime": 1568097678
+}
+---- QUERY
+# Test relative collection table refs. Ensure that the vertex ids of sources are
+# always an absolute explicit path.
+select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col m
+ where a.item = m.f1
+====
+---- LINEAGE
+{
+ "hash": "f6e042b9788a1583e212cf90c616dfd3",
+ "timestamp": 1568097683,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "ab",
+ "id": 0
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_array_col.item",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.value.f1",
+ "id": 2,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "c",
+ "id": 3
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.map_map_col.value.value",
+ "id": 4,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.map_map_col.value.key",
+ "id": 5,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "d",
+ "id": 6
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.int_map_col.value",
+ "id": 7,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "e",
+ "id": 8
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.value.f21",
+ "id": 9,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.complex_nested_struct_col.f2.item.f12.key",
+ "id": 10,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.allcomplextypes.struct_map_col.key",
+ "id": 11,
+ "metadata": {
+ "tableCreateTime": 1568093741,
+ "tableName": "functional.allcomplextypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1,
+ 2
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 4
+ ],
+ "targets": [
+ 3
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 5
+ ],
+ "targets": [
+ 3
+ ],
+ "edgeType": "PREDICATE"
+ },
+ {
+ "sources": [
+ 1,
+ 7
+ ],
+ "targets": [
+ 6
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 9
+ ],
+ "targets": [
+ 8
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1,
+ 10,
+ 11
+ ],
+ "targets": [
+ 0,
+ 3,
+ 6,
+ 8
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "c64f53b771511f69:143fbcbe00000000",
+ "user": "bharath",
+ "queryText": "select a + b as ab, c, d, e from functional.allcomplextypes t, (select sum(item) a from t.int_array_col where item < 10) v1, (select count(f1) b from t.struct_map_col group by key) v2, (select avg(value) over(partition by key) c from t.map_map_col.value) v3, (select item d from t.int_array_col union all select value from t.int_map_col) v4, (select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5",
+ "endTime": 1568097684
+}
+---- QUERY
+# Test relative collection table refs in inline views.
+select a + b as ab, c, d, e from functional.allcomplextypes t,
+ (select sum(item) a from t.int_array_col
+ where item < 10) v1,
+ (select count(f1) b from t.struct_map_col
+ group by key) v2,
+ (select avg(value) over(partition by key) c from t.map_map_col.value) v3,
+ (select item d from t.int_array_col
+ union all
+ select value from t.int_map_col) v4,
+ (select f21 e from t.complex_nested_struct_col.f2.f12 order by key limit 10) v5
+====
+---- LINEAGE
+{
+ "hash": "a0b32156f73829f9d72d35d5e6091684",
+ "timestamp": 1568097689,
+ "vertices": [
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "id",
+ "id": 0,
+ "metadata": {
+ "tableCreateTime": 1568097689,
+ "tableName": "lineage_test_db.lineage_test_view5"
+ }
+ },
+ {
+ "vertexType": "COLUMN",
+ "vertexId": "functional.alltypes.id",
+ "id": 1,
+ "metadata": {
+ "tableCreateTime": 1568093675,
+ "tableName": "functional.alltypes"
+ }
+ }
+ ],
+ "edges": [
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PROJECTION"
+ },
+ {
+ "sources": [
+ 1
+ ],
+ "targets": [
+ 0
+ ],
+ "edgeType": "PREDICATE"
+ }
+ ],
+ "queryId": "0d41edb23b146ceb:b9565e1000000000",
+ "user": "bharath",
+ "queryText": "create view lineage_test_db.lineage_test_view5 as select id from functional.alltypes_view v where not exists (select 1 from functional.alltypes a where v.id = a.id)",
+ "endTime": 1568097689
+}
+---- QUERY
+# IMPALA-4206: Test creating a view whose definition has a subquery and a view reference.
+create view lineage_test_db.lineage_test_view5 as
+select id from functional.alltypes_view v
+where not exists (select 1 from functional.alltypes a where v.id = a.id)
+====
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 6665fee..65bc255 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -17,6 +17,7 @@
#
# The base class that should be used for almost all Impala tests
+import glob
import grp
import json
import logging
@@ -51,6 +52,7 @@ from tests.common.test_dimensions import (
load_table_info_dimension)
from tests.common.test_result_verifier import (
try_compile_regex,
+ verify_lineage,
verify_raw_results,
verify_runtime_profile)
from tests.common.test_vector import ImpalaTestDimension
@@ -120,6 +122,7 @@ COMMENT_LINES_REGEX = r'(?:\s*--.*\n)*'
SET_PATTERN = re.compile(
COMMENT_LINES_REGEX + r'\s*set\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=*', re.I)
METRICS_URL = 'http://localhost:25000/metrics?json'
+VARZ_URL = 'http://localhost:25000/varz?json'
GROUP_NAME = grp.getgrgid(pwd.getpwnam(getuser()).pw_gid).gr_name
@@ -326,6 +329,16 @@ class ImpalaTestSuite(BaseTestSuite):
assert response.status_code == requests.codes.ok
return json.loads(response.text)
+ def get_var_current_val(self, var):
+ """Returns the current value of a given Impalad flag variable."""
+ # Parse the /varz endpoint to get the flag information.
+ varz = self.get_debug_page(VARZ_URL)
+ assert 'flags' in varz.keys()
+ filtered_varz = filter(lambda flag: flag['name'] == var, varz['flags'])
+ assert len(filtered_varz) == 1
+ assert 'current' in filtered_varz[0].keys()
+ return filtered_varz[0]['current'].strip()
+
def get_metric(self, name):
"""Finds the metric with name 'name' and returns its value as an int."""
def iter_metrics(group):
@@ -524,6 +537,8 @@ class ImpalaTestSuite(BaseTestSuite):
sections = self.load_query_test_file(self.get_workload(), test_file_name,
encoding=encoding)
+ # Assumes that it is same across all the coordinators.
+ lineage_log_dir = self.get_var_current_val('lineage_event_log_dir')
for test_section in sections:
if 'HIVE_MAJOR_VERSION' in test_section:
needed_hive_major_version = int(test_section['HIVE_MAJOR_VERSION'])
@@ -622,6 +637,20 @@ class ImpalaTestSuite(BaseTestSuite):
if pytest.config.option.update_results:
test_section[rt_profile_info] = "".join(rt_profile)
+ if 'LINEAGE' in test_section:
+ # Lineage flusher thread runs every 5s by default and is not configurable. Wait
+ # for that period. (TODO) Get rid of this for faster test execution.
+ time.sleep(5)
+ current_query_lineage = self.get_query_lineage(result.query_id, lineage_log_dir)
+ assert current_query_lineage is not "",\
+ "No lineage found for query %s in dir %s" %\
+ (result.query_id, lineage_log_dir)
+ if pytest.config.option.update_results:
+ test_section['LINEAGE'] = json.dumps(current_query_lineage, indent=2,
+ separators=(',', ': '))
+ else:
+ verify_lineage(json.loads(test_section['LINEAGE']), current_query_lineage)
+
if 'DML_RESULTS' in test_section:
assert 'ERRORS' not in test_section
# The limit is specified to ensure the queries aren't unbounded. We shouldn't have
@@ -637,6 +666,28 @@ class ImpalaTestSuite(BaseTestSuite):
test_file_name.replace('/','_') + ".test")
write_test_file(output_file, sections, encoding=encoding)
+ def get_query_lineage(self, query_id, lineage_dir):
+ """Walks through the lineage files in lineage_dir to look for a given query_id.
+ This is an expensive operation is lineage_dir is large, so use carefully."""
+ assert lineage_dir and os.path.isdir(lineage_dir),\
+ "Invalid lineage dir %s" % (lineage_dir)
+ lineage_files = glob.glob(os.path.join(lineage_dir, 'impala_lineage_log_1.0*'))
+ assert len(lineage_files) > 0, "Directory %s is empty" % (lineage_dir)
+ # Sort by mtime. Optimized for most recently written lineages.
+ lineage_files.sort(key=lambda f: os.path.getmtime(f), reverse=True)
+ for f in lineage_files:
+ with open(f) as fd:
+ # A single file can contain a maxmimum of 5000 entries by default.
+ lines = fd.readlines()
+ for line in reversed(lines):
+ line = line.strip()
+ if len(line) == 0: continue
+ lineage = json.loads(line)
+ assert 'queryId' in lineage.keys()
+ if lineage['queryId'] == query_id:
+ return lineage
+ return ""
+
def execute_test_case_setup(self, setup_section, table_format):
"""
Executes a test case 'SETUP' section
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index a9ba7c7..df3da73 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -39,6 +39,13 @@ COLUMN_REGEX_PREFIX = re.compile(COLUMN_REGEX_PREFIX_PATTERN, re.I)
ROW_REGEX_PREFIX_PATTERN = 'row_regex:'
ROW_REGEX_PREFIX = re.compile(ROW_REGEX_PREFIX_PATTERN, re.I)
+# Json keys that are skipped during comparison of two lineage JSON objects.
+# Lineages contain keys like timestamps, query_ids etc that are not expected
+# to match with other lineages. This list maintains the keys that are skipped
+# during comparison.
+DEFAULT_LINEAGE_SKIP_KEYS = ['tableCreateTime', 'queryId', 'timestamp', 'endTime',
+ 'user']
+
# Represents a single test result (row set)
class QueryTestResult(object):
def __init__(self, result_list, column_types, column_labels, order_matters):
@@ -620,6 +627,21 @@ def verify_runtime_profile(expected, actual, update_section=False):
% (function, field, expected_value, actual_value, actual))
return updated_aggregations
+
+def verify_lineage(expected, actual, lineage_skip_json_keys=DEFAULT_LINEAGE_SKIP_KEYS):
+ """Compares the lineage JSON objects expected and actual."""
+ def recursive_sort(obj):
+ if isinstance(obj, dict):
+ return sorted((k, recursive_sort(v))
+ for k, v in obj.items() if k not in lineage_skip_json_keys)
+ if isinstance(obj, list):
+ return sorted(recursive_sort(x) for x in obj)
+ return obj
+ sort_expected = recursive_sort(expected)
+ sort_actual = recursive_sort(actual)
+ assert sort_expected == sort_actual,\
+ "Lineage mismatch. EXPECTED:\n%s\n\nACTUAL:\n %s\n" % (sort_expected, sort_actual)
+
def get_node_exec_options(profile_string, exec_node_id):
""" Return a list with all of the ExecOption strings for the given exec node id. """
results = []
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 62746cb..992acf4 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -33,11 +33,16 @@ LOG = logging.getLogger(__name__)
class TestLineage(CustomClusterTestSuite):
START_END_TIME_LINEAGE_LOG_DIR = tempfile.mkdtemp(prefix="start_end_time")
CREATE_TABLE_TIME_LINEAGE_LOG_DIR = tempfile.mkdtemp(prefix="create_table_time")
+ LINEAGE_TESTS_DIR = tempfile.mkdtemp(prefix="test_lineage")
@classmethod
def setup_class(cls):
super(TestLineage, cls).setup_class()
+ @classmethod
+ def get_workload(cls):
+ return 'functional-query'
+
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
.format(START_END_TIME_LINEAGE_LOG_DIR))
@@ -100,3 +105,14 @@ class TestLineage(CustomClusterTestSuite):
table_create_time = int(vertex["metadata"]["tableCreateTime"])
assert "{0}.lineage_test_tbl".format(unique_database) == table_name
assert table_create_time != -1
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
+ .format(LINEAGE_TESTS_DIR))
+ def test_lineage_output(self, vector):
+ try:
+ self.run_test_case('QueryTest/lineage', vector)
+ finally:
+ # Clean up the test database
+ db_cleanup = "drop database if exists lineage_test_db cascade"
+ self.execute_query_expect_success(self.client, db_cleanup)
diff --git a/tests/unittests/test_file_parser.py b/tests/unittests/test_file_parser.py
index 435cffe..54a00f7 100644
--- a/tests/unittests/test_file_parser.py
+++ b/tests/unittests/test_file_parser.py
@@ -32,6 +32,9 @@ s
'Hi'
---- TYPES
string
+---- LINEAGE
+test_lineage_str > 'foo' AND 'bar'
+multi_line
====
---- QUERY
SELECT 2
@@ -57,7 +60,7 @@ int
====
"""
-VALID_SECTIONS = ['QUERY', 'RESULTS', 'TYPES']
+VALID_SECTIONS = ['QUERY', 'RESULTS', 'TYPES', 'LINEAGE']
class TestTestFileParser(BaseTestSuite):
def test_valid_parse(self):
@@ -65,7 +68,8 @@ class TestTestFileParser(BaseTestSuite):
assert len(results) == 3
print results[0]
expected_results = {'QUERY': '# comment\nSELECT blah from Foo\ns\n',
- 'TYPES': 'string\n', 'RESULTS': "'Hi'\n"}
+ 'TYPES': 'string\n', 'RESULTS': "'Hi'\n",
+ 'LINEAGE': "test_lineage_str > 'foo' AND 'bar'\nmulti_line\n"}
assert results[0] == expected_results
def test_invalid_section(self):
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index 9d90a99..095312f 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -96,7 +96,7 @@ def parse_query_test_file(file_name, valid_section_names=None, encoding=None):
if section_names is None:
section_names = ['QUERY', 'HIVE_QUERY', 'RESULTS', 'TYPES', 'LABELS', 'SETUP',
'CATCH', 'ERRORS', 'USER', 'RUNTIME_PROFILE', 'SHELL', 'DML_RESULTS',
- 'DBAPI_RESULTS', 'HS2_TYPES', 'HIVE_MAJOR_VERSION']
+ 'DBAPI_RESULTS', 'HS2_TYPES', 'HIVE_MAJOR_VERSION', 'LINEAGE']
return parse_test_file(file_name, section_names, encoding=encoding,
skip_unknown_sections=False)