You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/11/16 22:12:25 UTC
incubator-impala git commit: IMPALA-6080: clean up table descriptor
handling
Repository: incubator-impala
Updated Branches:
refs/heads/master ae116b5bf -> 70919187c
IMPALA-6080: clean up table descriptor handling
* Add DescriptorTbl::CreateHdfsTableDescriptor to avoid having to
create an entire DescriptorTbl during INSERT finalization (when only
a descriptor for the output table is needed)
* Remove TQueryExecRequest.desc_tbl, there's already a home for it in
TQueryContext.desc_tbl
This required fixing a problem in the planner test infrastructure
where the TQueryCtx was reused for planning multiple times despite
being modified during planning.
This is based on Marcel Kornacker's coordinator cleanup
patch.
Testing:
Ran core tests.
Change-Id: Id427dab0c196b556bd8b2d64ec618403d5cbd4d6
Reviewed-on: http://gerrit.cloudera.org:8080/8330
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/70919187
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/70919187
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/70919187
Branch: refs/heads/master
Commit: 70919187c64cced4f805cc995a3bcf5e1fc9f02f
Parents: ae116b5
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Wed Oct 18 17:02:36 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Nov 16 21:43:27 2017 +0000
----------------------------------------------------------------------
be/src/runtime/coordinator.cc | 19 +--
be/src/runtime/descriptors.cc | 132 ++++++++++---------
be/src/runtime/descriptors.h | 25 ++--
common/thrift/Frontend.thrift | 23 ++--
.../org/apache/impala/service/Frontend.java | 2 +-
.../apache/impala/planner/PlannerTestBase.java | 25 ++--
6 files changed, 116 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70919187/be/src/runtime/coordinator.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index 02c0ca1..772da33 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -95,9 +95,6 @@ Status Coordinator::Exec() {
<< " stmt=" << request.query_ctx.client_request.stmt;
stmt_type_ = request.stmt_type;
query_ctx_ = request.query_ctx;
- // set descriptor table here globally
- // TODO: remove TQueryExecRequest.desc_tbl
- query_ctx_.__set_desc_tbl(request.desc_tbl);
query_ctx_.__set_request_pool(schedule_.request_pool());
query_profile_ =
@@ -556,14 +553,9 @@ Status Coordinator::FinalizeSuccessfulInsert() {
// INSERT finalization happens in the five following steps
// 1. If OVERWRITE, remove all the files in the target directory
// 2. Create all the necessary partition directories.
- DescriptorTbl* descriptor_table;
- // TODO: add DescriptorTbl::CreateTableDescriptor() so we can create a
- // descriptor for just the output table, calling Create() can be very
- // expensive.
- RETURN_IF_ERROR(
- DescriptorTbl::Create(obj_pool(), query_ctx_.desc_tbl, &descriptor_table));
- HdfsTableDescriptor* hdfs_table = static_cast<HdfsTableDescriptor*>(
- descriptor_table->GetTableDescriptor(finalize_params_.table_id));
+ HdfsTableDescriptor* hdfs_table;
+ RETURN_IF_ERROR(DescriptorTbl::CreateHdfsTblDescriptor(query_ctx_.desc_tbl,
+ finalize_params_.table_id, obj_pool(), &hdfs_table));
DCHECK(hdfs_table != nullptr)
<< "INSERT target table not known in descriptor table: "
<< finalize_params_.table_id;
@@ -671,9 +663,8 @@ Status Coordinator::FinalizeSuccessfulInsert() {
}
}
- // Release resources on the descriptor table.
- descriptor_table->ReleaseResources();
- descriptor_table = nullptr;
+ // We're done with the HDFS descriptor - free up its resources.
+ hdfs_table->ReleaseResources();
hdfs_table = nullptr;
{
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70919187/be/src/runtime/descriptors.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/descriptors.cc b/be/src/runtime/descriptors.cc
index 6a4993b..bd67689 100644
--- a/be/src/runtime/descriptors.cc
+++ b/be/src/runtime/descriptors.cc
@@ -101,16 +101,15 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc,
tuple_offset_(tdesc.byteOffset),
null_indicator_offset_(tdesc.nullIndicatorByte, tdesc.nullIndicatorBit),
slot_idx_(tdesc.slotIdx),
- slot_size_(type_.GetSlotSize()),
- llvm_field_idx_(-1) {
+ slot_size_(type_.GetSlotSize()) {
DCHECK_NE(type_.type, TYPE_STRUCT);
- DCHECK(parent_ != NULL) << tdesc.parent;
+ DCHECK(parent_ != nullptr) << tdesc.parent;
if (type_.IsCollectionType()) {
DCHECK(tdesc.__isset.itemTupleId);
- DCHECK(collection_item_descriptor_ != NULL) << tdesc.itemTupleId;
+ DCHECK(collection_item_descriptor_ != nullptr) << tdesc.itemTupleId;
} else {
DCHECK(!tdesc.__isset.itemTupleId);
- DCHECK(collection_item_descriptor == NULL);
+ DCHECK(collection_item_descriptor == nullptr);
}
}
@@ -133,7 +132,7 @@ string SlotDescriptor::DebugString() const {
out << col_path_[i];
}
out << "]";
- if (collection_item_descriptor_ != NULL) {
+ if (collection_item_descriptor_ != nullptr) {
out << " collection_item_tuple_id=" << collection_item_descriptor_->id();
}
out << " offset=" << tuple_offset_ << " null=" << null_indicator_offset_.DebugString()
@@ -230,6 +229,16 @@ HdfsTableDescriptor::HdfsTableDescriptor(const TTableDescriptor& tdesc, ObjectPo
avro_schema_ = tdesc.hdfsTable.__isset.avroSchema ? tdesc.hdfsTable.avroSchema : "";
}
+void HdfsTableDescriptor::ReleaseResources() {
+ for (const auto& part_entry: partition_descriptors_) {
+ for (ScalarExprEvaluator* eval :
+ part_entry.second->partition_key_value_evals()) {
+ eval->Close(nullptr);
+ const_cast<ScalarExpr&>(eval->root()).Close();
+ }
+ }
+}
+
string HdfsTableDescriptor::DebugString() const {
stringstream out;
out << "HdfsTable(" << TableDescriptor::DebugString()
@@ -291,11 +300,9 @@ string KuduTableDescriptor::DebugString() const {
TupleDescriptor::TupleDescriptor(const TTupleDescriptor& tdesc)
: id_(tdesc.id),
- table_desc_(NULL),
byte_size_(tdesc.byteSize),
num_null_bytes_(tdesc.numNullBytes),
null_bytes_offset_(tdesc.byteSize - tdesc.numNullBytes),
- slots_(),
has_varlen_slots_(false),
tuple_path_(tdesc.tuplePath) {
}
@@ -506,31 +513,53 @@ Status DescriptorTbl::CreatePartKeyExprs(
return Status::OK();
}
+Status DescriptorTbl::CreateHdfsTblDescriptor(const TDescriptorTable& thrift_tbl,
+ TableId tbl_id, ObjectPool* pool, HdfsTableDescriptor** desc) {
+ for (const TTableDescriptor& tdesc: thrift_tbl.tableDescriptors) {
+ if (tdesc.id == tbl_id) {
+ DCHECK(tdesc.__isset.hdfsTable);
+ RETURN_IF_ERROR(CreateTblDescriptorInternal(
+ tdesc, pool, reinterpret_cast<TableDescriptor**>(desc)));
+ return Status::OK();
+ }
+ }
+ string error = Substitute("table $0 not found in descriptor table", tbl_id);
+ DCHECK(false) << error;
+ return Status(error);
+}
+
+Status DescriptorTbl::CreateTblDescriptorInternal(const TTableDescriptor& tdesc,
+ ObjectPool* pool, TableDescriptor** desc) {
+ *desc = nullptr;
+ switch (tdesc.tableType) {
+ case TTableType::HDFS_TABLE: {
+ HdfsTableDescriptor* hdfs_tbl = pool->Add(new HdfsTableDescriptor(tdesc, pool));
+ *desc = hdfs_tbl;
+ RETURN_IF_ERROR(CreatePartKeyExprs(*hdfs_tbl, pool));
+ break;
+ }
+ case TTableType::HBASE_TABLE:
+ *desc = pool->Add(new HBaseTableDescriptor(tdesc));
+ break;
+ case TTableType::DATA_SOURCE_TABLE:
+ *desc = pool->Add(new DataSourceTableDescriptor(tdesc));
+ break;
+ case TTableType::KUDU_TABLE:
+ *desc = pool->Add(new KuduTableDescriptor(tdesc));
+ break;
+ default:
+ DCHECK(false) << "invalid table type: " << tdesc.tableType;
+ }
+ return Status::OK();
+}
+
Status DescriptorTbl::Create(ObjectPool* pool, const TDescriptorTable& thrift_tbl,
DescriptorTbl** tbl) {
*tbl = pool->Add(new DescriptorTbl());
// deserialize table descriptors first, they are being referenced by tuple descriptors
- for (size_t i = 0; i < thrift_tbl.tableDescriptors.size(); ++i) {
- const TTableDescriptor& tdesc = thrift_tbl.tableDescriptors[i];
- TableDescriptor* desc = NULL;
- switch (tdesc.tableType) {
- case TTableType::HDFS_TABLE:
- desc = pool->Add(new HdfsTableDescriptor(tdesc, pool));
- RETURN_IF_ERROR(CreatePartKeyExprs(
- *static_cast<const HdfsTableDescriptor*>(desc), pool));
- break;
- case TTableType::HBASE_TABLE:
- desc = pool->Add(new HBaseTableDescriptor(tdesc));
- break;
- case TTableType::DATA_SOURCE_TABLE:
- desc = pool->Add(new DataSourceTableDescriptor(tdesc));
- break;
- case TTableType::KUDU_TABLE:
- desc = pool->Add(new KuduTableDescriptor(tdesc));
- break;
- default:
- DCHECK(false) << "invalid table type: " << tdesc.tableType;
- }
+ for (const TTableDescriptor& tdesc: thrift_tbl.tableDescriptors) {
+ TableDescriptor* desc;
+ RETURN_IF_ERROR(CreateTblDescriptorInternal(tdesc, pool, &desc));
(*tbl)->tbl_desc_map_[tdesc.id] = desc;
}
@@ -548,9 +577,9 @@ Status DescriptorTbl::Create(ObjectPool* pool, const TDescriptorTable& thrift_tb
const TSlotDescriptor& tdesc = thrift_tbl.slotDescriptors[i];
// Tuple descriptors are already populated in tbl
TupleDescriptor* parent = (*tbl)->GetTupleDescriptor(tdesc.parent);
- DCHECK(parent != NULL);
+ DCHECK(parent != nullptr);
TupleDescriptor* collection_item_descriptor = tdesc.__isset.itemTupleId ?
- (*tbl)->GetTupleDescriptor(tdesc.itemTupleId) : NULL;
+ (*tbl)->GetTupleDescriptor(tdesc.itemTupleId) : nullptr;
SlotDescriptor* slot_d = pool->Add(
new SlotDescriptor(tdesc, parent, collection_item_descriptor));
(*tbl)->slot_desc_map_[tdesc.id] = slot_d;
@@ -563,46 +592,23 @@ void DescriptorTbl::ReleaseResources() {
// close partition exprs of hdfs tables
for (auto entry: tbl_desc_map_) {
if (entry.second->type() != TTableType::HDFS_TABLE) continue;
- const HdfsTableDescriptor* hdfs_tbl =
- static_cast<const HdfsTableDescriptor*>(entry.second);
- for (const auto& part_entry: hdfs_tbl->partition_descriptors()) {
- for (ScalarExprEvaluator* eval :
- part_entry.second->partition_key_value_evals()) {
- eval->Close(nullptr);
- const_cast<ScalarExpr&>(eval->root()).Close();
- }
- }
+ static_cast<HdfsTableDescriptor*>(entry.second)->ReleaseResources();
}
}
TableDescriptor* DescriptorTbl::GetTableDescriptor(TableId id) const {
- // TODO: is there some boost function to do exactly this?
TableDescriptorMap::const_iterator i = tbl_desc_map_.find(id);
- if (i == tbl_desc_map_.end()) {
- return NULL;
- } else {
- return i->second;
- }
+ return i == tbl_desc_map_.end() ? nullptr : i->second;
}
TupleDescriptor* DescriptorTbl::GetTupleDescriptor(TupleId id) const {
- // TODO: is there some boost function to do exactly this?
TupleDescriptorMap::const_iterator i = tuple_desc_map_.find(id);
- if (i == tuple_desc_map_.end()) {
- return NULL;
- } else {
- return i->second;
- }
+ return i == tuple_desc_map_.end() ? nullptr : i->second;
}
SlotDescriptor* DescriptorTbl::GetSlotDescriptor(SlotId id) const {
- // TODO: is there some boost function to do exactly this?
SlotDescriptorMap::const_iterator i = slot_desc_map_.find(id);
- if (i == slot_desc_map_.end()) {
- return NULL;
- } else {
- return i->second;
- }
+ return i == slot_desc_map_.end() ? nullptr : i->second;
}
void DescriptorTbl::GetTupleDescs(vector<TupleDescriptor*>* descs) const {
@@ -627,7 +633,7 @@ llvm::Value* SlotDescriptor::CodegenIsNull(
llvm::Value* SlotDescriptor::CodegenIsNull(LlvmCodeGen* codegen, LlvmBuilder* builder,
const NullIndicatorOffset& null_indicator_offset, llvm::Value* tuple) {
llvm::Value* null_byte =
- CodegenGetNullByte(codegen, builder, null_indicator_offset, tuple, NULL);
+ CodegenGetNullByte(codegen, builder, null_indicator_offset, tuple, nullptr);
llvm::Constant* mask =
llvm::ConstantInt::get(codegen->tinyint_type(), null_indicator_offset.bit_mask);
llvm::Value* null_mask = builder->CreateAnd(null_byte, mask, "null_mask");
@@ -657,8 +663,8 @@ void SlotDescriptor::CodegenSetNullIndicator(
llvm::ConstantInt::get(codegen->tinyint_type(), ~null_indicator_offset_.bit_mask);
llvm::ConstantInt* constant_is_null = llvm::dyn_cast<llvm::ConstantInt>(is_null);
- llvm::Value* result = NULL;
- if (constant_is_null != NULL) {
+ llvm::Value* result = nullptr;
+ if (constant_is_null != nullptr) {
if (constant_is_null->isOne()) {
result = builder->CreateOr(null_byte, mask, "null_bit_set");
} else {
@@ -688,7 +694,7 @@ llvm::Value* SlotDescriptor::CodegenGetNullByte(
llvm::Value* tuple_bytes = builder->CreateBitCast(tuple, codegen->ptr_type());
llvm::Value* byte_ptr =
builder->CreateInBoundsGEP(tuple_bytes, byte_offset, "null_byte_ptr");
- if (null_byte_ptr != NULL) *null_byte_ptr = byte_ptr;
+ if (null_byte_ptr != nullptr) *null_byte_ptr = byte_ptr;
return builder->CreateLoad(byte_ptr, "null_byte");
}
@@ -707,7 +713,7 @@ llvm::StructType* TupleDescriptor::GetLlvmStruct(LlvmCodeGen* codegen) const {
int curr_struct_offset = 0;
for (SlotDescriptor* slot: sorted_slots) {
// IMPALA-3207: Codegen for CHAR is not yet implemented: bail out of codegen here.
- if (slot->type().type == TYPE_CHAR) return NULL;
+ if (slot->type().type == TYPE_CHAR) return nullptr;
DCHECK_EQ(curr_struct_offset, slot->tuple_offset());
slot->llvm_field_idx_ = struct_fields.size();
struct_fields.push_back(codegen->GetType(slot->type()));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70919187/be/src/runtime/descriptors.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 2e8a7c6..bca3ed6 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -52,6 +52,7 @@ class TDescriptorTable;
class TSlotDescriptor;
class TTable;
class TTupleDescriptor;
+class TTableDescriptor;
/// A path into a table schema (e.g. a vector of ColumnTypes) pointing to a particular
/// column/field. The i-th element of the path is the ordinal position of the column/field
@@ -77,12 +78,6 @@ class SchemaPathConstants {
DISALLOW_COPY_AND_ASSIGN(SchemaPathConstants);
};
-struct LlvmTupleStruct {
- llvm::StructType* tuple_struct;
- llvm::PointerType* tuple_ptr;
- std::vector<int> indices;
-};
-
/// Location information for null indicator bit for particular slot.
/// For non-nullable slots, the byte_offset will be 0 and the bit_mask will be 0.
/// This allows us to do the NullIndicatorOffset operations (tuple + byte_offset &/|
@@ -195,7 +190,7 @@ class SlotDescriptor {
/// The idx of the slot in the llvm codegen'd tuple struct
/// This is set by TupleDescriptor during codegen and takes into account
/// any padding bytes.
- int llvm_field_idx_;
+ int llvm_field_idx_ = -1;
/// collection_item_descriptor should be non-NULL iff this is a collection slot
SlotDescriptor(const TSlotDescriptor& tdesc, const TupleDescriptor* parent,
@@ -326,6 +321,9 @@ class HdfsTableDescriptor : public TableDescriptor {
return it->second;
}
+ /// Release resources by closing partition descriptors.
+ void ReleaseResources();
+
const PartitionIdToDescriptorMap& partition_descriptors() const {
return partition_descriptors_;
}
@@ -440,7 +438,7 @@ class TupleDescriptor {
friend class DescriptorTbl;
const TupleId id_;
- TableDescriptor* table_desc_;
+ TableDescriptor* table_desc_ = nullptr;
const int byte_size_;
const int num_null_bytes_;
const int null_bytes_offset_;
@@ -473,6 +471,12 @@ class TupleDescriptor {
class DescriptorTbl {
public:
+ /// Creates an HdfsTableDescriptor (allocated in 'pool' and returned via 'desc') for
+ /// table with id 'table_id' within thrift_tbl. DCHECKs if no such descriptor is
+ /// present.
+ static Status CreateHdfsTblDescriptor(const TDescriptorTable& thrift_tbl,
+ TableId table_id, ObjectPool* pool, HdfsTableDescriptor** desc);
+
/// Creates a descriptor tbl within 'pool' from thrift_tbl and returns it via 'tbl'.
/// Returns OK on success, otherwise error (in which case 'tbl' will be unset).
static Status Create(ObjectPool* pool, const TDescriptorTable& thrift_tbl,
@@ -503,6 +507,11 @@ class DescriptorTbl {
static Status CreatePartKeyExprs(
const HdfsTableDescriptor& hdfs_tbl, ObjectPool* pool) WARN_UNUSED_RESULT;
+
+ /// Creates a TableDescriptor (allocated in 'pool', returned via 'desc')
+ /// corresponding to tdesc. Returns error status on failure.
+ static Status CreateTblDescriptorInternal(const TTableDescriptor& tdesc,
+ ObjectPool* pool, TableDescriptor** desc);
};
/// Records positions of tuples within row produced by ExecNode. RowDescriptors are
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70919187/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index c874f6f..d684265 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -363,42 +363,39 @@ struct TPlanExecInfo {
// Result of call to ImpalaPlanService/JniFrontend.CreateQueryRequest()
struct TQueryExecRequest {
- // global descriptor tbl for all fragments
- 1: optional Descriptors.TDescriptorTable desc_tbl
-
// exec info for all plans; the first one materializes the query result
- 2: optional list<TPlanExecInfo> plan_exec_info
+ 1: optional list<TPlanExecInfo> plan_exec_info
// Metadata of the query result set (only for select)
- 3: optional Results.TResultSetMetadata result_set_metadata
+ 2: optional Results.TResultSetMetadata result_set_metadata
// Set if the query needs finalization after it executes
- 4: optional TFinalizeParams finalize_params
+ 3: optional TFinalizeParams finalize_params
- 5: required ImpalaInternalService.TQueryCtx query_ctx
+ 4: required ImpalaInternalService.TQueryCtx query_ctx
// The same as the output of 'explain <query>'
- 6: optional string query_plan
+ 5: optional string query_plan
// The statement type governs when the coordinator can judge a query to be finished.
// DML queries are complete after Wait(), SELECTs may not be. Generally matches
// the stmt_type of the parent TExecRequest, but in some cases (such as CREATE TABLE
// AS SELECT), these may differ.
- 7: required Types.TStmtType stmt_type
+ 6: required Types.TStmtType stmt_type
// List of replica hosts. Used by the host_idx field of TScanRangeLocation.
- 9: required list<Types.TNetworkAddress> host_list
+ 7: required list<Types.TNetworkAddress> host_list
// Column lineage graph
- 10: optional LineageGraph.TLineageGraph lineage_graph
+ 8: optional LineageGraph.TLineageGraph lineage_graph
// Estimated per-host peak memory consumption in bytes. Used by admission control.
// TODO: Remove when AC doesn't rely on this any more.
- 8: optional i64 per_host_mem_estimate
+ 9: optional i64 per_host_mem_estimate
// Maximum possible (in the case all fragments are scheduled on all hosts with
// max DOP) minimum reservation required per host, in bytes.
- 11: optional i64 max_per_host_min_reservation;
+ 10: optional i64 max_per_host_min_reservation;
}
enum TCatalogOpType {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70919187/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index f9a29f4..3f3a6d7 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -1119,7 +1119,7 @@ public class Frontend {
Planner planner = new Planner(analysisResult, queryCtx);
TQueryExecRequest queryExecRequest = createExecRequest(planner, explainString);
- queryExecRequest.setDesc_tbl(
+ queryCtx.setDesc_tbl(
planner.getAnalysisResult().getAnalyzer().getDescTbl().toThrift());
queryExecRequest.setQuery_ctx(queryCtx);
queryExecRequest.setHost_list(analysisResult.getAnalyzer().getHostIndex().getList());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/70919187/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
index 6932539..ab538ff 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
@@ -153,8 +153,8 @@ public class PlannerTestBase extends FrontendTestBase {
}
}
}
- if (execRequest.isSetDesc_tbl()) {
- TDescriptorTable descTbl = execRequest.desc_tbl;
+ if (execRequest.query_ctx.isSetDesc_tbl()) {
+ TDescriptorTable descTbl = execRequest.query_ctx.desc_tbl;
for (TTupleDescriptor tupleDesc: descTbl.tupleDescriptors) {
tupleMap_.put(tupleDesc.id, tupleDesc);
}
@@ -234,8 +234,9 @@ public class PlannerTestBase extends FrontendTestBase {
boolean first = true;
// Iterate through all partitions of the descriptor table and verify all partitions
// are referenced.
- if (execRequest.isSetDesc_tbl() && execRequest.desc_tbl.isSetTableDescriptors()) {
- for (TTableDescriptor tableDesc: execRequest.desc_tbl.tableDescriptors) {
+ if (execRequest.query_ctx.isSetDesc_tbl()
+ && execRequest.query_ctx.desc_tbl.isSetTableDescriptors()) {
+ for (TTableDescriptor tableDesc: execRequest.query_ctx.desc_tbl.tableDescriptors) {
// All partitions of insertTableId are okay.
if (tableDesc.getId() == insertTableId) continue;
if (!tableDesc.isSetHdfsTable()) continue;
@@ -410,22 +411,24 @@ public class PlannerTestBase extends FrontendTestBase {
throw new IllegalStateException("Cannot plan empty query in line: " +
testCase.getStartingLineNum());
}
+ // Set up the query context. Note that we need to deep copy it before planning each
+ // time since planning modifies it.
TQueryCtx queryCtx = TestUtils.createQueryContext(
dbName, System.getProperty("user.name"));
queryCtx.client_request.query_options = testCase.getOptions();
// Test single node plan, scan range locations, and column lineage.
- TExecRequest singleNodeExecRequest = testPlan(testCase, Section.PLAN, queryCtx,
+ TExecRequest singleNodeExecRequest = testPlan(testCase, Section.PLAN, queryCtx.deepCopy(),
ignoreExplainHeader, errorLog, actualOutput);
validateTableIds(singleNodeExecRequest);
checkScanRangeLocations(testCase, singleNodeExecRequest, errorLog, actualOutput);
checkColumnLineage(testCase, singleNodeExecRequest, errorLog, actualOutput);
checkLimitCardinality(query, singleNodeExecRequest, errorLog);
// Test distributed plan.
- testPlan(testCase, Section.DISTRIBUTEDPLAN, queryCtx, ignoreExplainHeader, errorLog,
- actualOutput);
+ testPlan(testCase, Section.DISTRIBUTEDPLAN, queryCtx.deepCopy(), ignoreExplainHeader,
+ errorLog, actualOutput);
// test parallel plans
- testPlan(testCase, Section.PARALLELPLANS, queryCtx, ignoreExplainHeader, errorLog,
- actualOutput);
+ testPlan(testCase, Section.PARALLELPLANS, queryCtx.deepCopy(), ignoreExplainHeader,
+ errorLog, actualOutput);
}
/**
@@ -436,8 +439,8 @@ public class PlannerTestBase extends FrontendTestBase {
if (request == null || !request.isSetQuery_exec_request()) return;
TQueryExecRequest execRequest = request.query_exec_request;
HashSet<Integer> seenTableIds = Sets.newHashSet();
- if (execRequest.isSetDesc_tbl()) {
- TDescriptorTable descTbl = execRequest.desc_tbl;
+ if (execRequest.query_ctx.isSetDesc_tbl()) {
+ TDescriptorTable descTbl = execRequest.query_ctx.desc_tbl;
if (descTbl.isSetTableDescriptors()) {
for (TTableDescriptor tableDesc: descTbl.tableDescriptors) {
if (seenTableIds.contains(tableDesc.id)) {