You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by lv...@apache.org on 2017/11/01 07:01:49 UTC
[2/2] incubator-impala git commit: IMPALA-5307: Part 2: copy out
strings in uncompressed Avro
IMPALA-5307: Part 2: copy out strings in uncompressed Avro
The approach is to re-materialize strings in those tuples that
survive conjunct evaluation and may reference disk I/O buffers
directly. This means that perf should not regress for the
following cases:
* Compressed Avro files.
* Non-string columns.
* Selective scans where the majority of tuples are filtered out.
This approach will also work for the Sequence and Text scanners.
Includes some improvements to Avro codegen to replace more constants to
help win back some performance (with limited success): replaced
InitTuple() with an optimised version and substituted
tuple_byte_size() with a constant.
Removes dead code for handling CHAR(n) - CHAR(n) is now always fixed
length.
Perf:
Did microbenchmarks on uncompressed Avro files, one with all columns
from lineitem and one with only l_comment. Tests were run with:
set num_scanner_threads=1;
I ran the query 5 times and extracted MaterializeTupleTime from the
profile to measure CPU cost of materialization. Overall string
materialization got significantly slower, mainly because of the
extra memcpy() calls required.
Selecting one string from a table with multiple columns:
select min(l_comment) from biglineitem_avro
1.814 -> 2.096
Selecting one string from a table with one column:
select min(l_comment) from biglineitem_comment; profile;
1.708 -> 3.7
Selecting one string from a table with one column with predicate:
select min(l_comment) from biglineitem_comment where length(l_comment) > 10000;
1.691 -> 1.449
Selecting all columns:
select min(l_orderkey), min(l_partkey), min(l_suppkey), min(l_linenumber),
min(l_quantity), min(l_extendedprice), min(l_discount), min(l_tax),
min(l_returnflag), min(l_linestatus), min(l_shipdate),
min(l_commitdate), min(l_receiptdate), min(l_shipinstruct),
min(l_shipmode), min(l_comment) from biglineitem_avro; profile;
2.335 -> 3.711
Selecting an int column (no strings):
select min(l_linenumber) from biglineitem_avro
1.806 -> 1.819
Testing:
Ran exhaustive tests.
Change-Id: If1fc78790d778c874f5aafa5958c3c045a88d233
Reviewed-on: http://gerrit.cloudera.org:8080/8146
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/10385c11
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/10385c11
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/10385c11
Branch: refs/heads/master
Commit: 10385c11a19218e711b074e49e7c2d394db4f354
Parents: efe95bd
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Sep 21 17:07:16 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Nov 1 04:26:44 2017 +0000
----------------------------------------------------------------------
be/src/codegen/gen_ir_descriptions.py | 8 +-
be/src/codegen/impala-ir.cc | 1 +
be/src/codegen/llvm-codegen.cc | 8 +-
be/src/codegen/llvm-codegen.h | 6 ++
be/src/common/status.cc | 2 +
be/src/common/status.h | 1 +
be/src/exec/hdfs-avro-scanner-ir.cc | 35 ++++---
be/src/exec/hdfs-avro-scanner.cc | 48 +++++++---
be/src/exec/hdfs-avro-scanner.h | 28 +++---
be/src/exec/hdfs-scan-node-base.h | 6 +-
be/src/exec/hdfs-scanner.cc | 53 ++++++++---
be/src/exec/hdfs-scanner.h | 46 +++++----
be/src/exec/scan-node.h | 2 +-
be/src/runtime/CMakeLists.txt | 1 +
be/src/runtime/descriptors.cc | 17 ++++
be/src/runtime/descriptors.h | 17 +++-
be/src/runtime/runtime-state.cc | 2 +
be/src/runtime/runtime-state.h | 2 +
be/src/runtime/tuple-ir.cc | 44 +++++++++
be/src/runtime/tuple.cc | 144 ++++++++++++++++++++++-------
be/src/runtime/tuple.h | 42 +++++++++
be/src/util/avro-util.h | 2 +
22 files changed, 401 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/codegen/gen_ir_descriptions.py
----------------------------------------------------------------------
diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py
index 5236f7b..1044a40 100755
--- a/be/src/codegen/gen_ir_descriptions.py
+++ b/be/src/codegen/gen_ir_descriptions.py
@@ -170,10 +170,12 @@ ir_functions = [
"_ZN6impala15HdfsAvroScanner12ReadAvroCharENS_13PrimitiveTypeEiPPhS2_bPvPNS_7MemPoolE"],
["READ_AVRO_DECIMAL",
"_ZN6impala15HdfsAvroScanner15ReadAvroDecimalEiPPhS1_bPvPNS_7MemPoolE"],
- ["HDFS_SCANNER_WRITE_ALIGNED_TUPLES",
- "_ZN6impala11HdfsScanner18WriteAlignedTuplesEPNS_7MemPoolEPNS_8TupleRowEiPNS_13FieldLocationEiiii"],
["HDFS_SCANNER_GET_CONJUNCT_EVALUATOR",
"_ZNK6impala11HdfsScanner15GetConjunctEvalEi"],
+ ["HDFS_SCANNER_INIT_TUPLE",
+ "_ZN6impala11HdfsScanner9InitTupleEPNS_5TupleES2_"],
+ ["HDFS_SCANNER_WRITE_ALIGNED_TUPLES",
+ "_ZN6impala11HdfsScanner18WriteAlignedTuplesEPNS_7MemPoolEPNS_8TupleRowEiPNS_13FieldLocationEiiii"],
["PROCESS_SCRATCH_BATCH",
"_ZN6impala18HdfsParquetScanner19ProcessScratchBatchEPNS_8RowBatchE"],
["PARQUET_SCANNER_EVAL_RUNTIME_FILTER",
@@ -203,6 +205,8 @@ ir_functions = [
"_ZN6impala7MemPool8AllocateILb1EEEPhli"],
["RUNTIME_FILTER_EVAL",
"_ZNK6impala13RuntimeFilter4EvalEPvRKNS_10ColumnTypeE"],
+ ["TUPLE_COPY_STRINGS",
+ "_ZN6impala5Tuple11CopyStringsEPKcPNS_12RuntimeStateEPKNS_11SlotOffsetsEiPNS_7MemPoolEPNS_6StatusE"],
["UNION_MATERIALIZE_BATCH",
"_ZN6impala9UnionNode16MaterializeBatchEPNS_8RowBatchEPPh"],
["BLOOM_FILTER_INSERT_NO_AVX2", "_ZN6impala11BloomFilter12InsertNoAvx2Ej"],
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/codegen/impala-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/impala-ir.cc b/be/src/codegen/impala-ir.cc
index bc3cfcb..2ae10a4 100644
--- a/be/src/codegen/impala-ir.cc
+++ b/be/src/codegen/impala-ir.cc
@@ -57,6 +57,7 @@
#include "runtime/mem-pool.h"
#include "runtime/raw-value-ir.cc"
#include "runtime/runtime-filter-ir.cc"
+#include "runtime/tuple-ir.cc"
#include "udf/udf-ir.cc"
#include "util/bloom-filter-ir.cc"
#include "util/hash-util-ir.cc"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/codegen/llvm-codegen.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc
index 31f7d9b..21678f2 100644
--- a/be/src/codegen/llvm-codegen.cc
+++ b/be/src/codegen/llvm-codegen.cc
@@ -1615,6 +1615,13 @@ Constant* LlvmCodeGen::ConstantToGVPtr(Type* type, Constant* ir_constant,
ArrayRef<Constant*>({GetIntConstant(TYPE_INT, 0)}));
}
+Constant* LlvmCodeGen::ConstantsToGVArrayPtr(Type* element_type,
+ ArrayRef<Constant*> ir_constants, const string& name) {
+ ArrayType* array_type = ArrayType::get(element_type, ir_constants.size());
+ Constant* array_const = ConstantArray::get(array_type, ir_constants);
+ return ConstantToGVPtr(array_type, array_const, name);
+}
+
void LlvmCodeGen::DiagnosticHandler::DiagnosticHandlerFn(const DiagnosticInfo &info,
void *context){
if (info.getSeverity() == DiagnosticSeverity::DS_Error) {
@@ -1637,7 +1644,6 @@ string LlvmCodeGen::DiagnosticHandler::GetErrorString() {
}
return "";
}
-
}
namespace boost {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/codegen/llvm-codegen.h
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.h b/be/src/codegen/llvm-codegen.h
index 072909f..c63f001 100644
--- a/be/src/codegen/llvm-codegen.h
+++ b/be/src/codegen/llvm-codegen.h
@@ -266,6 +266,12 @@ class LlvmCodeGen {
llvm::Constant* ConstantToGVPtr(llvm::Type* type, llvm::Constant* ir_constant,
const std::string& name);
+ /// Creates a global value 'name' that is an array with element type 'element_type'
+ /// containing 'ir_constants'. Returns a pointer to the global value, i.e. a pointer
+ /// to a constant array of 'element_type'.
+ llvm::Constant* ConstantsToGVArrayPtr(llvm::Type* element_type,
+ llvm::ArrayRef<llvm::Constant*> ir_constants, const std::string& name);
+
/// Returns reference to llvm context object. Each LlvmCodeGen has its own
/// context to allow multiple threads to be calling into llvm at the same time.
llvm::LLVMContext& context() { return *context_.get(); }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/common/status.cc
----------------------------------------------------------------------
diff --git a/be/src/common/status.cc b/be/src/common/status.cc
index 3fc2236..a009925 100644
--- a/be/src/common/status.cc
+++ b/be/src/common/status.cc
@@ -27,6 +27,8 @@
namespace impala {
+const char* Status::LLVM_CLASS_NAME = "class.impala::Status";
+
// NOTE: this is statically initialized and we must be very careful what
// functions these constructors call. In particular, we cannot call
// glog functions which also rely on static initializations.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/common/status.h
----------------------------------------------------------------------
diff --git a/be/src/common/status.h b/be/src/common/status.h
index 0f057e7..c3b8d68 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -249,6 +249,7 @@ class NODISCARD Status {
return msg_ == NULL ? TErrorCode::OK : msg_->error();
}
+ static const char* LLVM_CLASS_NAME;
private:
// Status constructors that can suppress logging via 'silent' parameter
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/hdfs-avro-scanner-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-avro-scanner-ir.cc b/be/src/exec/hdfs-avro-scanner-ir.cc
index 3c254cc..a2bf606 100644
--- a/be/src/exec/hdfs-avro-scanner-ir.cc
+++ b/be/src/exec/hdfs-avro-scanner-ir.cc
@@ -29,11 +29,15 @@ using std::numeric_limits;
// Functions in this file are cross-compiled to IR with clang.
-const int AVRO_FLOAT_SIZE = 4;
-const int AVRO_DOUBLE_SIZE = 8;
+static const int AVRO_FLOAT_SIZE = 4;
+static const int AVRO_DOUBLE_SIZE = 8;
int HdfsAvroScanner::DecodeAvroData(int max_tuples, MemPool* pool, uint8_t** data,
uint8_t* data_end, Tuple* tuple, TupleRow* tuple_row) {
+ // If the file is uncompressed, StringValues will have pointers into the I/O buffers.
+ // We don't attach I/O buffers to output batches so need to copy out data referenced
+ // by tuples that survive conjunct evaluation.
+ const bool copy_out_strings = !header_->is_compressed && !string_slot_offsets_.empty();
int num_to_commit = 0;
for (int i = 0; i < max_tuples; ++i) {
InitTuple(template_tuple_, tuple);
@@ -43,9 +47,16 @@ int HdfsAvroScanner::DecodeAvroData(int max_tuples, MemPool* pool, uint8_t** dat
}
tuple_row->SetTuple(scan_node_->tuple_idx(), tuple);
if (EvalConjuncts(tuple_row)) {
+ if (copy_out_strings) {
+ if (UNLIKELY(!tuple->CopyStrings("HdfsAvroScanner::DecodeAvroData()",
+ state_, string_slot_offsets_.data(), string_slot_offsets_.size(), pool,
+ &parse_status_))) {
+ return 0;
+ }
+ }
++num_to_commit;
tuple_row = next_row(tuple_row);
- tuple = next_tuple(tuple_byte_size_, tuple);
+ tuple = next_tuple(tuple_byte_size(), tuple);
}
}
return num_to_commit;
@@ -210,22 +221,8 @@ bool HdfsAvroScanner::ReadAvroChar(PrimitiveType type, int max_len, uint8_t** da
// We need to be careful not to truncate the length before evaluating min().
int str_len = static_cast<int>(std::min<int64_t>(len.val, max_len));
DCHECK_GE(str_len, 0);
- if (ctype.IsVarLenStringType()) {
- StringValue* sv = reinterpret_cast<StringValue*>(slot);
- sv->ptr = reinterpret_cast<char*>(pool->TryAllocate(max_len));
- if (UNLIKELY(sv->ptr == nullptr)) {
- string details = Substitute("HdfsAvroScanner::ReadAvroChar() failed to allocate"
- "$0 bytes for char slot.", max_len);
- parse_status_ = pool->mem_tracker()->MemLimitExceeded(state_, details, max_len);
- return false;
- }
- sv->len = max_len;
- memcpy(sv->ptr, *data, str_len);
- StringValue::PadWithSpaces(sv->ptr, max_len, str_len);
- } else {
- memcpy(slot, *data, str_len);
- StringValue::PadWithSpaces(reinterpret_cast<char*>(slot), max_len, str_len);
- }
+ memcpy(slot, *data, str_len);
+ StringValue::PadWithSpaces(reinterpret_cast<char*>(slot), max_len, str_len);
}
*data += len.val;
return true;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/hdfs-avro-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-avro-scanner.cc b/be/src/exec/hdfs-avro-scanner.cc
index 9254825..cad74e6 100644
--- a/be/src/exec/hdfs-avro-scanner.cc
+++ b/be/src/exec/hdfs-avro-scanner.cc
@@ -66,6 +66,10 @@ static Status CheckSchema(const AvroSchemaElement& avro_schema) {
HdfsAvroScanner::HdfsAvroScanner(HdfsScanNodeBase* scan_node, RuntimeState* state)
: BaseSequenceScanner(scan_node, state) {
+ for (SlotDescriptor* string_slot : scan_node_->tuple_desc()->string_slots()) {
+ string_slot_offsets_.push_back(
+ {string_slot->null_indicator_offset(), string_slot->tuple_offset()});
+ }
}
HdfsAvroScanner::HdfsAvroScanner()
@@ -76,6 +80,7 @@ HdfsAvroScanner::HdfsAvroScanner()
Status HdfsAvroScanner::Open(ScannerContext* context) {
RETURN_IF_ERROR(BaseSequenceScanner::Open(context));
RETURN_IF_ERROR(CheckSchema(scan_node_->avro_schema()));
+ stream_->set_contains_tuple_data(false); // Avro scanner always copies out data.
return Status::OK();
}
@@ -85,11 +90,7 @@ Status HdfsAvroScanner::Codegen(HdfsScanNodeBase* node,
DCHECK(node->runtime_state()->ShouldCodegen());
LlvmCodeGen* codegen = node->runtime_state()->codegen();
DCHECK(codegen != nullptr);
- Function* materialize_tuple_fn = nullptr;
- RETURN_IF_ERROR(CodegenMaterializeTuple(node, codegen, &materialize_tuple_fn));
- DCHECK(materialize_tuple_fn != nullptr);
- RETURN_IF_ERROR(CodegenDecodeAvroData(codegen, materialize_tuple_fn, conjuncts,
- decode_avro_data_fn));
+ RETURN_IF_ERROR(CodegenDecodeAvroData(node, codegen, conjuncts, decode_avro_data_fn));
DCHECK(*decode_avro_data_fn != nullptr);
return Status::OK();
}
@@ -760,7 +761,7 @@ void HdfsAvroScanner::SetStatusValueOverflow(TErrorCode::type error_code, int64_
// ret i1 false
// }
Status HdfsAvroScanner::CodegenMaterializeTuple(
- HdfsScanNodeBase* node, LlvmCodeGen* codegen, Function** materialize_tuple_fn) {
+ const HdfsScanNodeBase* node, LlvmCodeGen* codegen, Function** materialize_tuple_fn) {
LLVMContext& context = codegen->context();
LlvmBuilder builder(context);
@@ -889,11 +890,11 @@ Status HdfsAvroScanner::CodegenMaterializeTuple(
return Status::OK();
}
-Status HdfsAvroScanner::CodegenReadRecord(
- const SchemaPath& path, const AvroSchemaElement& record, int child_start,
- int child_end, HdfsScanNodeBase* node, LlvmCodeGen* codegen, void* void_builder,
- Function* fn, BasicBlock* insert_before, BasicBlock* bail_out, Value* this_val,
- Value* pool_val, Value* tuple_val, Value* data_val, Value* data_end_val) {
+Status HdfsAvroScanner::CodegenReadRecord(const SchemaPath& path,
+ const AvroSchemaElement& record, int child_start, int child_end,
+ const HdfsScanNodeBase* node, LlvmCodeGen* codegen, void* void_builder, Function* fn,
+ BasicBlock* insert_before, BasicBlock* bail_out, Value* this_val, Value* pool_val,
+ Value* tuple_val, Value* data_val, Value* data_end_val) {
RETURN_IF_ERROR(CheckSchema(record));
DCHECK_EQ(record.schema->type, AVRO_RECORD);
LLVMContext& context = codegen->context();
@@ -1062,15 +1063,23 @@ Status HdfsAvroScanner::CodegenReadScalar(const AvroSchemaElement& element,
return Status::OK();
}
-Status HdfsAvroScanner::CodegenDecodeAvroData(LlvmCodeGen* codegen,
- Function* materialize_tuple_fn, const vector<ScalarExpr*>& conjuncts,
+Status HdfsAvroScanner::CodegenDecodeAvroData(const HdfsScanNodeBase* node,
+ LlvmCodeGen* codegen, const vector<ScalarExpr*>& conjuncts,
Function** decode_avro_data_fn) {
SCOPED_TIMER(codegen->codegen_timer());
+
+ Function* materialize_tuple_fn;
+ RETURN_IF_ERROR(CodegenMaterializeTuple(node, codegen, &materialize_tuple_fn));
DCHECK(materialize_tuple_fn != nullptr);
Function* fn = codegen->GetFunction(IRFunction::DECODE_AVRO_DATA, true);
- int replaced = codegen->ReplaceCallSites(fn, materialize_tuple_fn, "MaterializeTuple");
+ Function* init_tuple_fn;
+ RETURN_IF_ERROR(CodegenInitTuple(node, codegen, &init_tuple_fn));
+ int replaced = codegen->ReplaceCallSites(fn, init_tuple_fn, "InitTuple");
+ DCHECK_EQ(replaced, 1);
+
+ replaced = codegen->ReplaceCallSites(fn, materialize_tuple_fn, "MaterializeTuple");
DCHECK_EQ(replaced, 1);
Function* eval_conjuncts_fn;
@@ -1079,6 +1088,17 @@ Status HdfsAvroScanner::CodegenDecodeAvroData(LlvmCodeGen* codegen,
replaced = codegen->ReplaceCallSites(fn, eval_conjuncts_fn, "EvalConjuncts");
DCHECK_EQ(replaced, 1);
+ Function* copy_strings_fn;
+ RETURN_IF_ERROR(Tuple::CodegenCopyStrings(
+ codegen, *node->tuple_desc(), ©_strings_fn));
+ replaced = codegen->ReplaceCallSites(fn, copy_strings_fn, "CopyStrings");
+ DCHECK_EQ(replaced, 1);
+
+ int tuple_byte_size = node->tuple_desc()->byte_size();
+ replaced = codegen->ReplaceCallSitesWithValue(fn,
+ codegen->GetIntConstant(TYPE_INT, tuple_byte_size), "tuple_byte_size");
+ DCHECK_EQ(replaced, 1);
+
fn->setName("DecodeAvroData");
*decode_avro_data_fn = codegen->FinalizeFunction(fn);
if (*decode_avro_data_fn == nullptr) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/hdfs-avro-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-avro-scanner.h b/be/src/exec/hdfs-avro-scanner.h
index 472d6d5..ab5746a 100644
--- a/be/src/exec/hdfs-avro-scanner.h
+++ b/be/src/exec/hdfs-avro-scanner.h
@@ -130,6 +130,12 @@ class HdfsAvroScanner : public BaseSequenceScanner {
bool use_codegend_decode_avro_data;
};
+ /// Offsets of string slots in the result tuple that may need to be copied as part of
+ /// tuple materialization. Populated in constructor. This is redundant with offset
+ /// information stored in the TupleDescriptor but storing only the required metadata
+ /// in a simple array of struct simplifies codegen and speeds up interpretation.
+ std::vector<SlotOffsets> string_slot_offsets_;
+
AvroFileHeader* avro_header_ = nullptr;
/// Current data block after decompression with its end and length.
@@ -208,18 +214,16 @@ class HdfsAvroScanner : public BaseSequenceScanner {
/// Produces a version of DecodeAvroData that uses codegen'd instead of interpreted
/// functions. Stores the resulting function in 'decode_avro_data_fn' if codegen was
/// successful or returns an error.
- static Status CodegenDecodeAvroData(LlvmCodeGen* codegen,
- llvm::Function* materialize_tuple_fn,
+ static Status CodegenDecodeAvroData(const HdfsScanNodeBase* node, LlvmCodeGen* codegen,
const std::vector<ScalarExpr*>& conjuncts,
- llvm::Function** decode_avro_data_fn)
- WARN_UNUSED_RESULT;
+ llvm::Function** decode_avro_data_fn) WARN_UNUSED_RESULT;
/// Codegens a version of MaterializeTuple() that reads records based on the table
/// schema. Stores the resulting function in 'materialize_tuple_fn' if codegen was
/// successful or returns an error.
/// TODO: Codegen a function for each unique file schema.
- static Status CodegenMaterializeTuple(HdfsScanNodeBase* node, LlvmCodeGen* codegen,
- llvm::Function** materialize_tuple_fn) WARN_UNUSED_RESULT;
+ static Status CodegenMaterializeTuple(const HdfsScanNodeBase* node,
+ LlvmCodeGen* codegen, llvm::Function** materialize_tuple_fn) WARN_UNUSED_RESULT;
/// Used by CodegenMaterializeTuple to recursively create the IR for reading an Avro
/// record.
@@ -237,12 +241,12 @@ class HdfsAvroScanner : public BaseSequenceScanner {
/// MaterializeTuple()
/// - child_start / child_end: specifies to only generate a subset of the record
/// schema's children
- static Status CodegenReadRecord(
- const SchemaPath& path, const AvroSchemaElement& record, int child_start,
- int child_end, HdfsScanNodeBase* node, LlvmCodeGen* codegen, void* builder,
- llvm::Function* fn, llvm::BasicBlock* insert_before, llvm::BasicBlock* bail_out,
- llvm::Value* this_val, llvm::Value* pool_val, llvm::Value* tuple_val,
- llvm::Value* data_val, llvm::Value* data_end_val) WARN_UNUSED_RESULT;
+ static Status CodegenReadRecord(const SchemaPath& path, const AvroSchemaElement& record,
+ int child_start, int child_end, const HdfsScanNodeBase* node, LlvmCodeGen* codegen,
+ void* builder, llvm::Function* fn, llvm::BasicBlock* insert_before,
+ llvm::BasicBlock* bail_out, llvm::Value* this_val, llvm::Value* pool_val,
+ llvm::Value* tuple_val, llvm::Value* data_val,
+ llvm::Value* data_end_val) WARN_UNUSED_RESULT;
/// Creates the IR for reading an Avro scalar at builder's current insert point.
static Status CodegenReadScalar(const AvroSchemaElement& element,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/hdfs-scan-node-base.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.h b/be/src/exec/hdfs-scan-node-base.h
index 252964b..f6e010a 100644
--- a/be/src/exec/hdfs-scan-node-base.h
+++ b/be/src/exec/hdfs-scan-node-base.h
@@ -151,10 +151,10 @@ class HdfsScanNodeBase : public ScanNode {
const TupleDescriptor* min_max_tuple_desc() const { return min_max_tuple_desc_; }
const TupleDescriptor* tuple_desc() const { return tuple_desc_; }
- const HdfsTableDescriptor* hdfs_table() { return hdfs_table_; }
- const AvroSchemaElement& avro_schema() { return *avro_schema_.get(); }
+ const HdfsTableDescriptor* hdfs_table() const { return hdfs_table_; }
+ const AvroSchemaElement& avro_schema() const { return *avro_schema_.get(); }
int skip_header_line_count() const { return skip_header_line_count_; }
- DiskIoRequestContext* reader_context() { return reader_context_; }
+ DiskIoRequestContext* reader_context() const { return reader_context_; }
bool optimize_parquet_count_star() const { return optimize_parquet_count_star_; }
int parquet_count_star_slot_offset() const { return parquet_count_star_slot_offset_; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/hdfs-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner.cc b/be/src/exec/hdfs-scanner.cc
index cae6004..4107e42 100644
--- a/be/src/exec/hdfs-scanner.cc
+++ b/be/src/exec/hdfs-scanner.cc
@@ -293,7 +293,7 @@ bool HdfsScanner::WriteCompleteTuple(MemPool* pool, FieldLocation* fields,
// eval_fail: ; preds = %parse
// ret i1 false
// }
-Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
+Status HdfsScanner::CodegenWriteCompleteTuple(const HdfsScanNodeBase* node,
LlvmCodeGen* codegen, const vector<ScalarExpr*>& conjuncts,
Function** write_complete_tuple_fn) {
*write_complete_tuple_fn = NULL;
@@ -368,23 +368,19 @@ Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
// Extract the input args
Value* this_arg = args[0];
Value* fields_arg = args[2];
- Value* tuple_arg = builder.CreateBitCast(args[3], tuple_ptr_type, "tuple_ptr");
+ Value* opaque_tuple_arg = args[3];
+ Value* tuple_arg = builder.CreateBitCast(opaque_tuple_arg, tuple_ptr_type, "tuple_ptr");
Value* tuple_row_arg = args[4];
- Value* template_arg = builder.CreateBitCast(args[5], tuple_ptr_type, "tuple_ptr");
+ Value* opaque_template_arg = args[5];
Value* errors_arg = args[6];
Value* error_in_row_arg = args[7];
// Codegen for function body
Value* error_in_row = codegen->false_value();
- // Initialize tuple
- if (node->num_materialized_partition_keys() == 0) {
- // No partition key slots, just zero the NULL bytes.
- codegen->CodegenClearNullBits(&builder, tuple_arg, *tuple_desc);
- } else {
- // Copy template tuple.
- // TODO: only copy what's necessary from the template tuple.
- codegen->CodegenMemcpy(&builder, tuple_arg, template_arg, tuple_desc->byte_size());
- }
+
+ Function* init_tuple_fn;
+ RETURN_IF_ERROR(CodegenInitTuple(node, codegen, &init_tuple_fn));
+ builder.CreateCall(init_tuple_fn, {this_arg, opaque_template_arg, opaque_tuple_arg});
// Put tuple in tuple_row
Value* tuple_row_typed =
@@ -513,7 +509,7 @@ Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
return Status::OK();
}
-Status HdfsScanner::CodegenWriteAlignedTuples(HdfsScanNodeBase* node,
+Status HdfsScanner::CodegenWriteAlignedTuples(const HdfsScanNodeBase* node,
LlvmCodeGen* codegen, Function* write_complete_tuple_fn,
Function** write_aligned_tuples_fn) {
*write_aligned_tuples_fn = NULL;
@@ -535,6 +531,37 @@ Status HdfsScanner::CodegenWriteAlignedTuples(HdfsScanNodeBase* node,
return Status::OK();
}
+Status HdfsScanner::CodegenInitTuple(
+ const HdfsScanNodeBase* node, LlvmCodeGen* codegen, Function** init_tuple_fn) {
+ *init_tuple_fn = codegen->GetFunction(IRFunction::HDFS_SCANNER_INIT_TUPLE, true);
+ DCHECK(*init_tuple_fn != nullptr);
+
+ // Replace all of the constants in InitTuple() to specialize the code.
+ int replaced = codegen->ReplaceCallSitesWithBoolConst(
+ *init_tuple_fn, node->num_materialized_partition_keys() > 0, "has_template_tuple");
+ DCHECK_EQ(replaced, 1);
+
+ const TupleDescriptor* tuple_desc = node->tuple_desc();
+ replaced = codegen->ReplaceCallSitesWithValue(*init_tuple_fn,
+ codegen->GetIntConstant(TYPE_INT, tuple_desc->byte_size()), "tuple_byte_size");
+ DCHECK_EQ(replaced, 1);
+
+ replaced = codegen->ReplaceCallSitesWithValue(*init_tuple_fn,
+ codegen->GetIntConstant(TYPE_INT, tuple_desc->null_bytes_offset()),
+ "null_bytes_offset");
+ DCHECK_EQ(replaced, 1);
+
+ replaced = codegen->ReplaceCallSitesWithValue(*init_tuple_fn,
+ codegen->GetIntConstant(TYPE_INT, tuple_desc->num_null_bytes()), "num_null_bytes");
+ DCHECK_EQ(replaced, 1);
+
+ *init_tuple_fn = codegen->FinalizeFunction(*init_tuple_fn);
+ if (*init_tuple_fn == nullptr) {
+ return Status("Failed to finalize codegen'd InitTuple().");
+ }
+ return Status::OK();
+}
+
Status HdfsScanner::UpdateDecompressor(const THdfsCompression::type& compression) {
// Check whether the file in the stream has different compression from the last one.
if (compression != decompression_type_) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/hdfs-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner.h b/be/src/exec/hdfs-scanner.h
index 2005870..cb69bbe 100644
--- a/be/src/exec/hdfs-scanner.h
+++ b/be/src/exec/hdfs-scanner.h
@@ -390,33 +390,44 @@ class HdfsScanner {
/// Codegen function to replace WriteCompleteTuple. Should behave identically
/// to WriteCompleteTuple. Stores the resulting function in 'write_complete_tuple_fn'
/// if codegen was successful or NULL otherwise.
- static Status CodegenWriteCompleteTuple(HdfsScanNodeBase* node, LlvmCodeGen* codegen,
- const std::vector<ScalarExpr*>& conjuncts,
- llvm::Function** write_complete_tuple_fn)
- WARN_UNUSED_RESULT;
+ static Status CodegenWriteCompleteTuple(const HdfsScanNodeBase* node,
+ LlvmCodeGen* codegen, const std::vector<ScalarExpr*>& conjuncts,
+ llvm::Function** write_complete_tuple_fn) WARN_UNUSED_RESULT;
/// Codegen function to replace WriteAlignedTuples. WriteAlignedTuples is cross
/// compiled to IR. This function loads the precompiled IR function, modifies it,
/// and stores the resulting function in 'write_aligned_tuples_fn' if codegen was
/// successful or NULL otherwise.
- static Status CodegenWriteAlignedTuples(HdfsScanNodeBase*, LlvmCodeGen*,
- llvm::Function* write_tuple_fn, llvm::Function** write_aligned_tuples_fn)
- WARN_UNUSED_RESULT;
+ static Status CodegenWriteAlignedTuples(const HdfsScanNodeBase*, LlvmCodeGen*,
+ llvm::Function* write_tuple_fn,
+ llvm::Function** write_aligned_tuples_fn) WARN_UNUSED_RESULT;
+
+ /// Codegen function to replace InitTuple() removing runtime constants like the tuple
+ /// size and branches like the template tuple existence check. The codegen'd version
+ /// of InitTuple() is stored in 'init_tuple_fn' if codegen was successful.
+ static Status CodegenInitTuple(
+ const HdfsScanNodeBase* node, LlvmCodeGen* codegen, llvm::Function** init_tuple_fn);
/// Report parse error for column @ desc. If abort_on_error is true, sets
/// parse_status_ to the error message.
void ReportColumnParseError(const SlotDescriptor* desc, const char* data, int len);
- /// Initialize a tuple.
- /// TODO: only copy over non-null slots.
- void InitTuple(const TupleDescriptor* desc, Tuple* template_tuple, Tuple* tuple) {
- if (template_tuple != NULL) {
- InitTupleFromTemplate(template_tuple, tuple, desc->byte_size());
+ /// Initialize a tuple. Inlined into the convenience version below for codegen.
+ void IR_ALWAYS_INLINE InitTuple(
+ const TupleDescriptor* desc, Tuple* template_tuple, Tuple* tuple) {
+ if (has_template_tuple()) {
+ InitTupleFromTemplate(template_tuple, tuple, tuple_byte_size());
} else {
- tuple->ClearNullBits(*desc);
+ tuple->ClearNullBits(desc->null_bytes_offset(), desc->num_null_bytes());
}
}
+ /// Convenience version of above that passes in the scan's TupleDescriptor.
+ /// Replaced with a codegen'd version in IR.
+ void IR_NO_INLINE InitTuple(Tuple* template_tuple, Tuple* tuple) {
+ return InitTuple(scan_node_->tuple_desc(), template_tuple, tuple);
+ }
+
/// Initialize 'tuple' with size 'tuple_byte_size' from 'template_tuple'
void InitTupleFromTemplate(Tuple* template_tuple, Tuple* tuple, int tuple_byte_size) {
memcpy(tuple, template_tuple, tuple_byte_size);
@@ -454,9 +465,12 @@ class HdfsScanner {
}
}
- void InitTuple(Tuple* template_tuple, Tuple* tuple) {
- InitTuple(scan_node_->tuple_desc(), template_tuple, tuple);
- }
+ /// Not inlined in IR so it can be replaced with a constant.
+ int IR_NO_INLINE tuple_byte_size() const { return tuple_byte_size_; }
+
+ /// Returns true iff there is a template tuple with partition key values.
+ /// Not inlined in IR so it can be replaced with a constant.
+ bool IR_NO_INLINE has_template_tuple() const { return template_tuple_ != nullptr; }
inline Tuple* next_tuple(int tuple_byte_size, Tuple* t) const {
uint8_t* mem = reinterpret_cast<uint8_t*>(t);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/exec/scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/scan-node.h b/be/src/exec/scan-node.h
index ff79857..0976e27 100644
--- a/be/src/exec/scan-node.h
+++ b/be/src/exec/scan-node.h
@@ -101,7 +101,7 @@ class ScanNode : public ExecNode {
virtual bool IsScanNode() const { return true; }
- RuntimeState* runtime_state() { return runtime_state_; }
+ RuntimeState* runtime_state() const { return runtime_state_; }
RuntimeProfile::Counter* bytes_read_counter() const { return bytes_read_counter_; }
RuntimeProfile::Counter* rows_read_counter() const { return rows_read_counter_; }
RuntimeProfile::Counter* collection_items_read_counter() const {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index faed531..4d95dda 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -67,6 +67,7 @@ add_library(Runtime
timestamp-parse-util.cc
timestamp-value.cc
tuple.cc
+ tuple-ir.cc
tuple-row.cc
tmp-file-mgr.cc
)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/descriptors.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/descriptors.cc b/be/src/runtime/descriptors.cc
index ef553fb..7f86145 100644
--- a/be/src/runtime/descriptors.cc
+++ b/be/src/runtime/descriptors.cc
@@ -37,6 +37,11 @@
#include "common/names.h"
using boost::algorithm::join;
+using llvm::Constant;
+using llvm::ConstantAggregateZero;
+using llvm::ConstantInt;
+using llvm::ConstantStruct;
+using llvm::StructType;
using namespace strings;
// In 'thrift_partition', the location is stored in a compressed format that references
@@ -65,6 +70,7 @@ namespace impala {
const int RowDescriptor::INVALID_IDX;
const char* TupleDescriptor::LLVM_CLASS_NAME = "class.impala::TupleDescriptor";
+const char* NullIndicatorOffset::LLVM_CLASS_NAME = "struct.impala::NullIndicatorOffset";
string NullIndicatorOffset::DebugString() const {
stringstream out;
@@ -73,6 +79,17 @@ string NullIndicatorOffset::DebugString() const {
return out.str();
}
+Constant* NullIndicatorOffset::ToIR(LlvmCodeGen* codegen) const {
+ StructType* null_indicator_offset_type =
+ static_cast<StructType*>(codegen->GetType(LLVM_CLASS_NAME));
+ // Populate padding at end of struct with zeroes.
+ ConstantAggregateZero* zeroes = ConstantAggregateZero::get(null_indicator_offset_type);
+ return ConstantStruct::get(null_indicator_offset_type,
+ {ConstantInt::get(codegen->int_type(), byte_offset),
+ ConstantInt::get(codegen->tinyint_type(), bit_mask),
+ zeroes->getStructElement(2)});
+}
+
ostream& operator<<(ostream& os, const NullIndicatorOffset& null_indicator) {
os << null_indicator.DebugString();
return os;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/descriptors.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 572bfda..2e8a7c6 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -24,6 +24,7 @@
#include <boost/scoped_ptr.hpp>
#include <ostream>
+#include "codegen/impala-ir.h"
#include "common/global-types.h"
#include "common/status.h"
#include "runtime/types.h"
@@ -32,6 +33,7 @@
#include "gen-cpp/Types_types.h"
namespace llvm {
+ class Constant;
class Function;
class PointerType;
class StructType;
@@ -86,6 +88,9 @@ struct LlvmTupleStruct {
/// This allows us to do the NullIndicatorOffset operations (tuple + byte_offset &/|
/// bit_mask) regardless of whether the slot is nullable or not.
/// This is more efficient than branching to check if the slot is non-nullable.
+///
+/// ToIR() generates a constant version of this struct in LLVM IR. If the struct
+/// layout is updated, then ToIR() must also be updated.
struct NullIndicatorOffset {
int byte_offset;
uint8_t bit_mask; /// to extract null indicator
@@ -100,6 +105,12 @@ struct NullIndicatorOffset {
}
std::string DebugString() const;
+
+ // Generates an LLVM IR constant of this offset. Needs to be updated if the layout of
+ // this struct changes.
+ llvm::Constant* ToIR(LlvmCodeGen* codegen) const;
+
+ static const char* LLVM_CLASS_NAME;
};
std::ostream& operator<<(std::ostream& os, const NullIndicatorOffset& null_indicator);
@@ -387,8 +398,10 @@ class KuduTableDescriptor : public TableDescriptor {
class TupleDescriptor {
public:
int byte_size() const { return byte_size_; }
- int num_null_bytes() const { return num_null_bytes_; }
- int null_bytes_offset() const { return null_bytes_offset_; }
+ // num_null_bytes() and null_bytes_offset() are not inlined so they can be replaced with
+ // constants during codegen.
+ int IR_NO_INLINE num_null_bytes() const { return num_null_bytes_; }
+ int IR_NO_INLINE null_bytes_offset() const { return null_bytes_offset_; }
const std::vector<SlotDescriptor*>& slots() const { return slots_; }
const std::vector<SlotDescriptor*>& string_slots() const { return string_slots_; }
const std::vector<SlotDescriptor*>& collection_slots() const {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index be393f1..ac08c04 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -60,6 +60,8 @@ DECLARE_int32(max_errors);
namespace impala {
+const char* RuntimeState::LLVM_CLASS_NAME = "class.impala::RuntimeState";
+
RuntimeState::RuntimeState(QueryState* query_state, const TPlanFragmentCtx& fragment_ctx,
const TPlanFragmentInstanceCtx& instance_ctx, ExecEnv* exec_env)
: query_state_(query_state),
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 3da9f8c..c8ae147 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -305,6 +305,8 @@ class RuntimeState {
/// Release resources and prepare this object for destruction. Can only be called once.
void ReleaseResources();
+ static const char* LLVM_CLASS_NAME;
+
private:
/// Allow TestEnv to use private methods for testing.
friend class TestEnv;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/tuple-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tuple-ir.cc b/be/src/runtime/tuple-ir.cc
new file mode 100644
index 0000000..fb3dfbd
--- /dev/null
+++ b/be/src/runtime/tuple-ir.cc
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/tuple.h"
+
+#include "runtime/string-value.h"
+
+namespace impala {
+
+bool Tuple::CopyStrings(const char* err_ctx, RuntimeState* state,
+ const SlotOffsets* string_slot_offsets, int num_string_slots, MemPool* pool,
+ Status* status) noexcept {
+ int64_t total_len = 0;
+ for (int i = 0; i < num_string_slots; ++i) {
+ if (IsNull(string_slot_offsets[i].null_indicator_offset)) continue;
+ total_len += GetStringSlot(string_slot_offsets[i].tuple_offset)->len;
+ }
+ char* buf = AllocateStrings(err_ctx, state, total_len, pool, status);
+ if (UNLIKELY(buf == nullptr)) return false;
+ for (int i = 0; i < num_string_slots; ++i) {
+ if (IsNull(string_slot_offsets[i].null_indicator_offset)) continue;
+ StringValue* sv = GetStringSlot(string_slot_offsets[i].tuple_offset);
+ int str_len = sv->len;
+ memcpy(buf, sv->ptr, str_len);
+ sv->ptr = buf;
+ buf += str_len;
+ }
+ return true;
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/tuple.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tuple.cc b/be/src/runtime/tuple.cc
index 1dab5a5..cab7686 100644
--- a/be/src/runtime/tuple.cc
+++ b/be/src/runtime/tuple.cc
@@ -22,11 +22,13 @@
#include "codegen/codegen-anyval.h"
#include "codegen/llvm-codegen.h"
-#include "exprs/scalar-expr.h"
#include "exprs/scalar-expr-evaluator.h"
+#include "exprs/scalar-expr.h"
+#include "gutil/strings/substitute.h"
#include "runtime/collection-value.h"
#include "runtime/descriptors.h"
#include "runtime/mem-pool.h"
+#include "runtime/mem-tracker.h"
#include "runtime/raw-value.h"
#include "runtime/runtime-state.h"
#include "runtime/string-value.h"
@@ -36,11 +38,10 @@
#include "common/names.h"
-using namespace llvm;
-
namespace impala {
const char* Tuple::LLVM_CLASS_NAME = "class.impala::Tuple";
+const char* SlotOffsets::LLVM_CLASS_NAME = "struct.impala::SlotOffsets";
const char* Tuple::MATERIALIZE_EXPRS_SYMBOL = "MaterializeExprsILb0ELb0";
const char* Tuple::MATERIALIZE_EXPRS_NULL_POOL_SYMBOL = "MaterializeExprsILb0ELb1";
@@ -232,6 +233,18 @@ void Tuple::MaterializeExprs(TupleRow* row, const TupleDescriptor& desc,
}
}
+char* Tuple::AllocateStrings(const char* err_ctx, RuntimeState* state,
+ int64_t bytes, MemPool* pool, Status* status) noexcept {
+ char* buf = reinterpret_cast<char*>(pool->TryAllocateUnaligned(bytes));
+ if (UNLIKELY(buf == nullptr)) {
+ string details = Substitute("$0 failed to allocate $1 bytes for strings.",
+ err_ctx, bytes);
+ *status = pool->mem_tracker()->MemLimitExceeded(state, details, bytes);
+ return nullptr;
+ }
+ return buf;
+}
+
// Codegens an unrolled version of MaterializeExprs(). Uses codegen'd exprs and slot
// writes. If 'pool' is non-NULL, string data is copied into it.
//
@@ -291,19 +304,19 @@ void Tuple::MaterializeExprs(TupleRow* row, const TupleDescriptor& desc,
// }
Status Tuple::CodegenMaterializeExprs(LlvmCodeGen* codegen, bool collect_string_vals,
const TupleDescriptor& desc, const vector<ScalarExpr*>& slot_materialize_exprs,
- bool use_mem_pool, Function** fn) {
+ bool use_mem_pool, llvm::Function** fn) {
// Only support 'collect_string_vals' == false for now.
if (collect_string_vals) {
return Status("CodegenMaterializeExprs() collect_string_vals == true NYI");
}
SCOPED_TIMER(codegen->codegen_timer());
- LLVMContext& context = codegen->context();
+ llvm::LLVMContext& context = codegen->context();
// Codegen each compute function from slot_materialize_exprs
- Function* materialize_expr_fns[slot_materialize_exprs.size()];
+ llvm::Function* materialize_expr_fns[slot_materialize_exprs.size()];
for (int i = 0; i < slot_materialize_exprs.size(); ++i) {
- Status status = slot_materialize_exprs[i]->GetCodegendComputeFn(codegen,
- &materialize_expr_fns[i]);
+ Status status = slot_materialize_exprs[i]->GetCodegendComputeFn(
+ codegen, &materialize_expr_fns[i]);
if (!status.ok()) {
return Status::Expected(Substitute("Could not codegen CodegenMaterializeExprs: $0",
status.GetDetail()));
@@ -315,15 +328,15 @@ Status Tuple::CodegenMaterializeExprs(LlvmCodeGen* codegen, bool collect_string_
// void MaterializeExprs(Tuple* tuple, TupleRow* row, TupleDescriptor* desc,
// ScalarExprEvaluator** slot_materialize_exprs, MemPool* pool,
// StringValue** non_null_string_values, int* total_string_lengths)
- PointerType* opaque_tuple_type = codegen->GetPtrType(Tuple::LLVM_CLASS_NAME);
- PointerType* row_type = codegen->GetPtrType(TupleRow::LLVM_CLASS_NAME);
- PointerType* desc_type = codegen->GetPtrType(TupleDescriptor::LLVM_CLASS_NAME);
- PointerType* expr_evals_type =
+ llvm::PointerType* opaque_tuple_type = codegen->GetPtrType(Tuple::LLVM_CLASS_NAME);
+ llvm::PointerType* row_type = codegen->GetPtrType(TupleRow::LLVM_CLASS_NAME);
+ llvm::PointerType* desc_type = codegen->GetPtrType(TupleDescriptor::LLVM_CLASS_NAME);
+ llvm::PointerType* expr_evals_type =
codegen->GetPtrType(codegen->GetPtrType(ScalarExprEvaluator::LLVM_CLASS_NAME));
- PointerType* pool_type = codegen->GetPtrType(MemPool::LLVM_CLASS_NAME);
- PointerType* string_values_type =
+ llvm::PointerType* pool_type = codegen->GetPtrType(MemPool::LLVM_CLASS_NAME);
+ llvm::PointerType* string_values_type =
codegen->GetPtrType(codegen->GetPtrType(StringValue::LLVM_CLASS_NAME));
- PointerType* int_ptr_type = codegen->GetPtrType(TYPE_INT);
+ llvm::PointerType* int_ptr_type = codegen->GetPtrType(TYPE_INT);
LlvmCodeGen::FnPrototype prototype(codegen, "MaterializeExprs", codegen->void_type());
prototype.AddArgument("opaque_tuple", opaque_tuple_type);
prototype.AddArgument("row", row_type);
@@ -335,25 +348,25 @@ Status Tuple::CodegenMaterializeExprs(LlvmCodeGen* codegen, bool collect_string_
prototype.AddArgument("num_non_null_string_values", int_ptr_type);
LlvmBuilder builder(context);
- Value* args[8];
+ llvm::Value* args[8];
*fn = prototype.GeneratePrototype(&builder, args);
- Value* opaque_tuple_arg = args[0];
- Value* row_arg = args[1];
- // Value* desc_arg = args[2]; // unused
- Value* expr_evals_arg = args[3];
- Value* pool_arg = args[4];
+ llvm::Value* opaque_tuple_arg = args[0];
+ llvm::Value* row_arg = args[1];
+ // llvm::Value* desc_arg = args[2]; // unused
+ llvm::Value* expr_evals_arg = args[3];
+ llvm::Value* pool_arg = args[4];
// The followings arguments are unused as 'collect_string_vals' is false.
- // Value* non_null_string_values_arg = args[5]; // unused
- // Value* total_string_lengths_arg = args[6]; // unused
- // Value* num_non_null_string_values_arg = args[7]; // unused
+ // llvm::Value* non_null_string_values_arg = args[5]; // unused
+ // llvm::Value* total_string_lengths_arg = args[6]; // unused
+ // llvm::Value* num_non_null_string_values_arg = args[7]; // unused
// Cast the opaque Tuple* argument to the generated struct type
- Type* tuple_struct_type = desc.GetLlvmStruct(codegen);
+ llvm::Type* tuple_struct_type = desc.GetLlvmStruct(codegen);
if (tuple_struct_type == NULL) {
return Status("CodegenMaterializeExprs(): failed to generate tuple desc");
}
- PointerType* tuple_type = codegen->GetPtrType(tuple_struct_type);
- Value* tuple = builder.CreateBitCast(opaque_tuple_arg, tuple_type, "tuple");
+ llvm::PointerType* tuple_type = codegen->GetPtrType(tuple_struct_type);
+ llvm::Value* tuple = builder.CreateBitCast(opaque_tuple_arg, tuple_type, "tuple");
// Clear tuple's null bytes
codegen->CodegenClearNullBits(&builder, tuple, desc);
@@ -361,13 +374,13 @@ Status Tuple::CodegenMaterializeExprs(LlvmCodeGen* codegen, bool collect_string_
// Evaluate the slot_materialize_exprs and place the results in the tuple.
for (int i = 0; i < desc.slots().size(); ++i) {
SlotDescriptor* slot_desc = desc.slots()[i];
- DCHECK(slot_desc->type().type == TYPE_NULL ||
- slot_desc->type() == slot_materialize_exprs[i]->type());
+ DCHECK(slot_desc->type().type == TYPE_NULL
+ || slot_desc->type() == slot_materialize_exprs[i]->type());
// Call materialize_expr_fns[i](slot_materialize_exprs[i], row)
- Value* expr_eval =
+ llvm::Value* expr_eval =
codegen->CodegenArrayAt(&builder, expr_evals_arg, i, "expr_eval");
- Value* expr_args[] = { expr_eval, row_arg };
+ llvm::Value* expr_args[] = {expr_eval, row_arg};
CodegenAnyVal src = CodegenAnyVal::CreateCallWrapped(codegen, &builder,
slot_materialize_exprs[i]->type(), materialize_expr_fns[i], expr_args, "src");
@@ -378,9 +391,78 @@ Status Tuple::CodegenMaterializeExprs(LlvmCodeGen* codegen, bool collect_string_
// TODO: if pool != NULL, OptimizeFunctionWithExprs() is inlining the Allocate()
// call. Investigate if this is a good thing.
*fn = codegen->FinalizeFunction(*fn);
+ if (*fn == nullptr) {
+ return Status("Tuple::CodegenMaterializeTuple(): failed to finalize function");
+ }
return Status::OK();
}
+Status Tuple::CodegenCopyStrings(
+ LlvmCodeGen* codegen, const TupleDescriptor& desc, llvm::Function** copy_strings_fn) {
+ llvm::PointerType* opaque_tuple_type = codegen->GetPtrType(Tuple::LLVM_CLASS_NAME);
+ llvm::PointerType* runtime_state_type =
+ codegen->GetPtrType(RuntimeState::LLVM_CLASS_NAME);
+ llvm::StructType* slot_offsets_type =
+ static_cast<llvm::StructType*>(codegen->GetType(SlotOffsets::LLVM_CLASS_NAME));
+ llvm::PointerType* pool_type = codegen->GetPtrType(MemPool::LLVM_CLASS_NAME);
+ llvm::PointerType* status_type = codegen->GetPtrType(Status::LLVM_CLASS_NAME);
+ LlvmCodeGen::FnPrototype prototype(
+ codegen, "CopyStringsWrapper", codegen->boolean_type());
+ prototype.AddArgument("opaque_tuple", opaque_tuple_type);
+ prototype.AddArgument("err_ctx", codegen->ptr_type());
+ prototype.AddArgument("state", runtime_state_type);
+ prototype.AddArgument("slot_offsets", codegen->GetPtrType(slot_offsets_type));
+ prototype.AddArgument("num_string_slots", codegen->int_type());
+ prototype.AddArgument("pool", pool_type);
+ prototype.AddArgument("status", status_type);
+
+ LlvmBuilder builder(codegen->context());
+ llvm::Value* args[7];
+ *copy_strings_fn = prototype.GeneratePrototype(&builder, args);
+ llvm::Value* opaque_tuple_arg = args[0];
+ llvm::Value* err_ctx_arg = args[1];
+ llvm::Value* state_arg = args[2];
+ // slot_offsets and num_string_slots are replaced with constants so args are unused.
+ llvm::Value* pool_arg = args[5];
+ llvm::Value* status_arg = args[6];
+
+ llvm::Function* cross_compiled_fn =
+ codegen->GetFunction(IRFunction::TUPLE_COPY_STRINGS, false);
+ DCHECK(cross_compiled_fn != nullptr);
+
+ // Convert the offsets of the string slots into a constant IR array 'slot_offsets'.
+ vector<llvm::Constant*> slot_offset_ir_constants;
+ for (SlotDescriptor* slot_desc : desc.string_slots()) {
+ SlotOffsets offsets = {slot_desc->null_indicator_offset(), slot_desc->tuple_offset()};
+ slot_offset_ir_constants.push_back(offsets.ToIR(codegen));
+ }
+ llvm::Constant* constant_slot_offsets = codegen->ConstantsToGVArrayPtr(
+ slot_offsets_type, slot_offset_ir_constants, "slot_offsets");
+ llvm::Constant* num_string_slots =
+ llvm::ConstantInt::get(codegen->int_type(), desc.string_slots().size());
+ // Get SlotOffsets* pointer to the first element of the constant array.
+ llvm::Value* constant_slot_offsets_first_element_ptr =
+ builder.CreateConstGEP2_64(constant_slot_offsets, 0, 0);
+
+ llvm::Value* result_val = builder.CreateCall(cross_compiled_fn,
+ {opaque_tuple_arg, err_ctx_arg, state_arg, constant_slot_offsets_first_element_ptr,
+ num_string_slots, pool_arg, status_arg});
+ builder.CreateRet(result_val);
+
+ *copy_strings_fn = codegen->FinalizeFunction(*copy_strings_fn);
+ if (*copy_strings_fn == nullptr) {
+ return Status("Tuple::CodegenCopyStrings(): failed to finalize function");
+ }
+ return Status::OK();
+}
+
+llvm::Constant* SlotOffsets::ToIR(LlvmCodeGen* codegen) const {
+ return llvm::ConstantStruct::get(
+ static_cast<llvm::StructType*>(codegen->GetType(LLVM_CLASS_NAME)),
+ {null_indicator_offset.ToIR(codegen),
+ llvm::ConstantInt::get(codegen->int_type(), tuple_offset)});
+}
+
template void Tuple::MaterializeExprs<false, false>(TupleRow*, const TupleDescriptor&,
ScalarExprEvaluator* const*, MemPool*, StringValue**, int*, int*);
template void Tuple::MaterializeExprs<false, true>(TupleRow*, const TupleDescriptor&,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/runtime/tuple.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/tuple.h b/be/src/runtime/tuple.h
index 5503ab4..3286f10 100644
--- a/be/src/runtime/tuple.h
+++ b/be/src/runtime/tuple.h
@@ -27,6 +27,7 @@
namespace llvm {
class Function;
+class Constant;
}
namespace impala {
@@ -36,6 +37,20 @@ struct StringValue;
class TupleDescriptor;
class TupleRow;
+/// Minimal struct to hold slot offset information from a SlotDescriptor. Designed
+/// to simplify constant substitution in CodegenCopyStrings() and allow more efficient
+/// interpretation in CopyStrings().
+struct SlotOffsets {
+ NullIndicatorOffset null_indicator_offset;
+ int tuple_offset;
+
+ /// Generate an LLVM Constant containing the offset values of this SlotOffsets instance.
+ /// Needs to be updated if the layout of this struct changes.
+ llvm::Constant* ToIR(LlvmCodeGen* codegen) const;
+
+ static const char* LLVM_CLASS_NAME;
+};
+
/// A tuple is stored as a contiguous sequence of bytes containing a fixed number
/// of fixed-size slots. The slots are arranged in order of increasing byte length;
/// the tuple might contain padding between slots in order to align them according
@@ -158,6 +173,17 @@ class Tuple {
if (COLLECT_STRING_VALS) non_null_string_values->resize(num_non_null_string_values);
}
+ /// Copy the var-len string data in this tuple into the provided memory pool and update
+ /// the string slots to point at the copied strings. 'string_slot_offsets' contains the
+ /// required offsets in a contiguous array to allow efficient iteration and easy
+ /// substitution of the array with constants for codegen. 'err_ctx' is a string that is
+ /// included in error messages to provide context. Returns true on success, otherwise on
+ /// failure sets 'status' to an error. This odd return convention is used to avoid
+ /// emitting unnecessary code for ~Status() in perf-critical code.
+ bool CopyStrings(const char* err_ctx, RuntimeState* state,
+ const SlotOffsets* string_slot_offsets, int num_string_slots, MemPool* pool,
+ Status* status) noexcept;
+
/// Symbols (or substrings of the symbols) of MaterializeExprs(). These can be passed to
/// LlvmCodeGen::ReplaceCallSites().
static const char* MATERIALIZE_EXPRS_SYMBOL;
@@ -176,6 +202,14 @@ class Tuple {
const TupleDescriptor& desc, const vector<ScalarExpr*>& slot_materialize_exprs,
bool use_mem_pool, llvm::Function** fn);
+ /// Generates an IR version of CopyStrings(). The offsets of string slots are replaced
+ /// with constants. This can allow the LLVM optimiser to unroll the loop and generate
+ /// efficient code without interpretation overhead. The LLVM optimiser at -O2 generally
+ /// will only unroll the loop for small numbers of iterations: 8 or less in some
+ /// experiements. On success, 'fn' is set to point to the generated function.
+ static Status CodegenCopyStrings(LlvmCodeGen* codegen,
+ const TupleDescriptor& desc, llvm::Function** materialize_strings_fn);
+
/// Turn null indicator bit on. For non-nullable slots, the mask will be 0 and
/// this is a no-op (but we don't have to branch to check is slots are nullable).
void SetNull(const NullIndicatorOffset& offset) {
@@ -250,6 +284,14 @@ class Tuple {
ScalarExprEvaluator* const* evals, MemPool* pool,
StringValue** non_null_string_values, int* total_string_lengths,
int* num_non_null_string_values);
+
+ /// Helper for CopyStrings() to allocate 'bytes' of memory. Returns a pointer to the
+ /// allocated buffer on success. Otherwise an error was encountered, in which case NULL
+ /// is returned and 'status' is set to an error. This odd return convention is used to
+ /// avoid emitting unnecessary code for ~Status() in perf-critical code.
+ char* AllocateStrings(const char* err_ctx, RuntimeState* state, int64_t bytes,
+ MemPool* pool, Status* status) noexcept;
+
};
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/10385c11/be/src/util/avro-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/avro-util.h b/be/src/util/avro-util.h
index 3106279..d8283df 100644
--- a/be/src/util/avro-util.h
+++ b/be/src/util/avro-util.h
@@ -72,7 +72,9 @@ struct ScopedAvroSchemaElement {
~ScopedAvroSchemaElement();
AvroSchemaElement* operator->() { return &element_; }
+ const AvroSchemaElement* operator->() const { return &element_; }
AvroSchemaElement* get() { return &element_; }
+ const AvroSchemaElement* get() const { return &element_; }
private:
AvroSchemaElement element_;