You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2026/07/31 11:59:21 UTC
(impala) branch master updated (2115e498f -> f0d6958ff)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 2115e498f IMPALA-14600: Support HBO for AggregationNode cardinality
new 7fe7e1b35 IMPALA-15221: Harden VARIANT decoding against corrupt input
new f0d6958ff IMPALA-15193: [addendum] ST_Disjoint Incorrect Results Workaround
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exprs/variant-functions-ir.cc | 6 +-
be/src/runtime/types.h | 4 +
be/src/runtime/variant-value.cc | 493 ++++++++++++---------
be/src/runtime/variant-value.h | 127 ++++--
be/src/util/variant-util-test.cc | 211 ++++++++-
.../main/java/org/apache/impala/catalog/Type.java | 1 +
.../impala/hive/geospatial/esri/ST_Disjoint.java | 41 ++
.../queries/QueryTest/geospatial-esri-extra.test | 30 +-
8 files changed, 650 insertions(+), 263 deletions(-)
(impala) 02/02: IMPALA-15193: [addendum] ST_Disjoint Incorrect Results Workaround
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit f0d6958ff10de4a526e5b28b9b6411d452d5a17b
Author: jasonmfehr <jf...@cloudera.com>
AuthorDate: Thu Jul 30 15:02:04 2026 -0700
IMPALA-15193: [addendum] ST_Disjoint Incorrect Results Workaround
Workaround of a bug in the ST_Disjoint built-in UDF caused by a bug
in the underpinning esri-geometry-api library. When comparing two
geometries to determine if they are disjoint where the first
geometry is either a LineString or a MultiLineString and the second
geometry is a MultiLineString, then ST_Disjoint incorrectly returns
false when the first geometry is within the envelope of the second.
The first patch for this issue fixed the ST_Intersects function.
Since the underlying bug is in the disjoint predicate implementation
of the esri-geometry-api library, the workaround must also be
applied to the ST_Disjoint function implementation.
Tested via the addition of new query tests with the test results
double checked in postgis.
Assisted-by: Github Copilot (GPT-5.3-Codex)
Change-Id: I66c23c6db1140add91d86dbaeaec9884bcb6a831
Reviewed-on: http://gerrit.cloudera.org:8080/24648
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../impala/hive/geospatial/esri/ST_Disjoint.java | 41 ++++++++++++++++++++++
.../queries/QueryTest/geospatial-esri-extra.test | 30 +++++++++++++++-
2 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/ST_Disjoint.java b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/ST_Disjoint.java
index fa4d9152f..befc166ff 100644
--- a/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/ST_Disjoint.java
+++ b/java/hive-geospatial-functions/src/main/java/org/apache/impala/hive/geospatial/esri/ST_Disjoint.java
@@ -19,6 +19,11 @@ package org.apache.impala.hive.geospatial.esri;
import com.esri.core.geometry.OperatorDisjoint;
import com.esri.core.geometry.OperatorSimpleRelation;
+import com.esri.core.geometry.SpatialReference;
+import com.esri.core.geometry.ogc.OGCGeometry;
+import com.esri.core.geometry.ogc.OGCLineString;
+import com.esri.core.geometry.ogc.OGCMultiLineString;
+
import org.apache.hadoop.hive.ql.exec.Description;
@Description(name = "ST_Disjoint",
@@ -38,4 +43,40 @@ public class ST_Disjoint extends ST_GeometryRelational {
public String getDisplayString(String[] args) {
return String.format("returns true if %s and %s are disjoint", args[0], args[1]);
}
+
+ @Override
+ protected Object execEvaluate(OGCGeometry geom1, OGCGeometry geom2,
+ SpatialReference sr) {
+ // IMPALA-15193: Workaround for an underlying issue in the ESRI geometry-api-java
+ // dependency. This issue causes ST_Intersects to incorrectly return true when
+ // geometry 2 is a MultiLineString and geometry 1 is either a LineString or a
+ // MultiLineString.
+ if (geom2 instanceof OGCMultiLineString) {
+ if (geom1 instanceof OGCLineString) {
+ OGCMultiLineString geom2MLS = (OGCMultiLineString) geom2;
+ for (int i = 0; i < geom2MLS.numGeometries(); i++) {
+ if (! (Boolean) super.execEvaluate(geom1, geom2MLS.geometryN(i), sr)) {
+ return false;
+ }
+ }
+
+ return true;
+ } else if(geom1 instanceof OGCMultiLineString) {
+ OGCMultiLineString geom1MLS = (OGCMultiLineString) geom1;
+ OGCMultiLineString geom2MLS = (OGCMultiLineString) geom2;
+ for (int i = 0; i < geom1MLS.numGeometries(); i++) {
+ for (int j = 0; j < geom2MLS.numGeometries(); j++) {
+ if (! (Boolean) super.execEvaluate(geom1MLS.geometryN(i), geom2MLS.geometryN(j),
+ sr)) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+ }
+
+ return super.execEvaluate(geom1, geom2, sr);
+ }
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-extra.test b/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-extra.test
index a7dfbab8f..694bc17e4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-extra.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/geospatial-esri-extra.test
@@ -168,7 +168,7 @@ BIGINT
7393893
====
---- QUERY
-select ST_Intersects(st_geomfromtext('LINESTRING(1 0, 1 1)'), st_geomfromtext('MULTILINESTRING((0 0, 0 1), (2 0, 2 1))'));
+select st_intersects(st_geomfromtext('LINESTRING(1 0, 1 1)'), st_geomfromtext('MULTILINESTRING((0 0, 0 1), (2 0, 2 1))'));
---- TYPES
BOOLEAN
---- RESULTS
@@ -194,4 +194,32 @@ select st_intersects(st_geomfromtext('MULTILINESTRING((2 0, 2 2),(1 0, 1 1))'),
BOOLEAN
---- RESULTS
true
+====
+---- QUERY
+select st_disjoint(st_geomfromtext('LINESTRING(1 0, 1 1)'), st_geomfromtext('MULTILINESTRING((0 0, 0 1), (2 0, 2 1))'));
+---- TYPES
+BOOLEAN
+---- RESULTS
+true
+====
+---- QUERY
+select st_disjoint(st_geomfromtext('MULTILINESTRING((2 0, 2 2),(3 0, 3 2))'), st_geomfromtext('MULTILINESTRING((0 0, 0 2), (4 0, 4 2))'));
+---- TYPES
+BOOLEAN
+---- RESULTS
+true
+====
+---- QUERY
+select st_disjoint(st_geomfromtext('LINESTRING(1 0, 1 1)'), st_geomfromtext('MULTILINESTRING((-1 1, 2 1), (2 0, 2 1))'));
+---- TYPES
+BOOLEAN
+---- RESULTS
+false
+====
+---- QUERY
+select st_disjoint(st_geomfromtext('MULTILINESTRING((2 0, 2 2),(1 0, 1 1))'), st_geomfromtext('MULTILINESTRING((-1 1, 2 1), (2 0, 2 1))'));
+---- TYPES
+BOOLEAN
+---- RESULTS
+false
====
\ No newline at end of file
(impala) 01/02: IMPALA-15221: Harden VARIANT decoding against corrupt input
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7fe7e1b35a11e5db9f6bdad636bb4159fa08ee16
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Fri Jul 24 16:21:38 2026 +0200
IMPALA-15221: Harden VARIANT decoding against corrupt input
The VariantValue/VariantMetadata decoder added in IMPALA-15067 performed
almost no bounds checking on the offsets and lengths read from the value
and metadata blobs. Because VARIANT values can be written by other
engines into Iceberg/Parquet files, these blobs are untrusted input: a
corrupt or adversarial blob could cause reads outside the allocated
buffer: a crash, or information disclosure via the JSON conversion
This patch adds bounds checking throughout the decoder.
Metadata (validated once in VariantMetadata::Init):
- Compute the offset-array length in 64-bit so a corrupt (large)
dictionary size cannot overflow the length check.
- Record the string-data length and verify the dictionary offsets are
non-decreasing and stay within it, so GetFieldName() can never return
a string_view that points out of bounds.
Value (checked on access):
- Scalar accessors (GetInt*, GetFloat/GetDouble, GetString/GetBinary and
the ReadValue<T> helper) now return a bool and fail on a type mismatch
or a truncated payload instead of reading past the end of the buffer.
- Object/array navigation (GetFieldByIndex, GetArrayElement,
GetFieldByName, GetObjectSize, GetArraySize, GetFieldNameByIndex)
parses and bounds-checks the header and offset table before use,
validates each element's extent, and checks that field ids are within
the metadata dictionary.
- JSON serialization enforces a maximum nesting depth so a deeply nested
value cannot overflow the stack, and returns an error on any malformed
encoding.
Checking bounds at each access, keeps partial-access callers such as
variant_get() lazy and lets them share the same protection.
Testing:
- Added negative unit tests in variant-util-test.cc
- Existing tests pass
Change-Id: I3a22bb16cd05854a9afb56ebd27806ac52670bca
Assisted-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
Reviewed-on: http://gerrit.cloudera.org:8080/24635
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Peter Rozsa <pr...@cloudera.com>
---
be/src/exprs/variant-functions-ir.cc | 6 +-
be/src/runtime/types.h | 4 +
be/src/runtime/variant-value.cc | 493 ++++++++++++---------
be/src/runtime/variant-value.h | 127 ++++--
be/src/util/variant-util-test.cc | 211 ++++++++-
.../main/java/org/apache/impala/catalog/Type.java | 1 +
6 files changed, 580 insertions(+), 262 deletions(-)
diff --git a/be/src/exprs/variant-functions-ir.cc b/be/src/exprs/variant-functions-ir.cc
index 2e5684b89..42bae4ef8 100644
--- a/be/src/exprs/variant-functions-ir.cc
+++ b/be/src/exprs/variant-functions-ir.cc
@@ -27,7 +27,11 @@ StringVal VariantFunctions::VariantToJson(FunctionContext* ctx,
StringVal result;
Status status = impala::VariantToJson(
ctx, metadata.ptr, metadata.len, value.ptr, value.len, &result);
- if (!status.ok()) return StringVal::null();
+ if (!status.ok()) {
+ // A corrupt/unrepresentable variant value yields SQL NULL; surface why via a warning.
+ ctx->AddWarning(status.GetDetail().c_str());
+ return StringVal::null();
+ }
return result;
}
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index 5a6a73089..deaa3a114 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -98,6 +98,10 @@ struct ColumnType {
/// The maximum precision representable by a 8-byte decimal (Decimal8Value)
static const int MAX_DECIMAL8_PRECISION = 18;
+ /// Maximum nesting depth of a type or value. Must be kept in sync with FE's
+ /// Type.MAX_NESTING_DEPTH.
+ static const int MAX_NESTING_DEPTH = 100;
+
/// Empty for scalar types
std::vector<ColumnType> children;
diff --git a/be/src/runtime/variant-value.cc b/be/src/runtime/variant-value.cc
index cdbc1a3c7..0777dc99a 100644
--- a/be/src/runtime/variant-value.cc
+++ b/be/src/runtime/variant-value.cc
@@ -56,7 +56,9 @@ Status VariantMetadata::Init(const uint8_t* data, uint32_t len) {
is_sorted_ = (header >> 4) & 0x01;
offset_size_ = ((header >> 6) & 0x03) + 1;
- int pos = 1;
+ // All offset arithmetic below is done in 64-bit so that a corrupt (large) dict_size_
+ // cannot overflow the bounds checks and let an under-sized blob pass validation.
+ uint64_t pos = 1;
if (pos + offset_size_ > len) {
return Status("Variant metadata too short for dictionary size");
}
@@ -68,13 +70,33 @@ Status VariantMetadata::Init(const uint8_t* data, uint32_t len) {
// Offsets array: (dict_size_ + 1) entries of offset_size_ bytes each.
offsets_ = data + pos;
- int offsets_len = (dict_size_ + 1) * offset_size_;
+ uint64_t offsets_len = (static_cast<uint64_t>(dict_size_) + 1) * offset_size_;
if (pos + offsets_len > len) {
return Status("Variant metadata too short for offset array");
}
pos += offsets_len;
string_data_ = data + pos;
+ // pos <= len is guaranteed by the check above, so this fits in uint32_t.
+ string_data_len_ = static_cast<uint32_t>(len - pos);
+
+ // Validate the dictionary offsets once here (rather than on every field-name access):
+ // they must be non-decreasing and stay within the string-data region, so GetFieldName()
+ // can never construct a string_view that points out of bounds.
+ uint32_t prev = ReadOffset(0);
+ for (uint32_t i = 1; i <= dict_size_; ++i) {
+ uint32_t cur = ReadOffset(i);
+ // Offsets must be non-decreasing. Equal consecutive offsets denote a zero-length
+ // (empty-string) field name, e.g. the key in {"": 1}: that is valid JSON and is not
+ // forbidden by the Variant spec, so only a strict decrease is rejected.
+ if (cur < prev) {
+ return Status("Variant metadata has non-monotonic dictionary offsets");
+ }
+ prev = cur;
+ }
+ if (prev > string_data_len_) {
+ return Status("Variant metadata dictionary offset exceeds string data");
+ }
return Status::OK();
}
@@ -101,9 +123,13 @@ uint32_t VariantMetadata::ReadOffset(uint32_t index) const {
}
string_view VariantMetadata::GetFieldName(uint32_t index) const {
+ // Callers (FindFieldId and FieldNameFromLayout) must bound-check the index.
DCHECK_LT(index, dict_size_);
uint32_t start = ReadOffset(index);
uint32_t end = ReadOffset(index + 1);
+ // Guaranteed by the offset validation in Init().
+ DCHECK_LE(start, end);
+ DCHECK_LE(end, string_data_len_);
return string_view(reinterpret_cast<const char*>(string_data_ + start),
end - start);
}
@@ -163,69 +189,64 @@ bool VariantValue::IsNull() const {
&& GetPhysicalType() == VariantPhysicalType::VNULL;
}
-bool VariantValue::GetBoolean() const {
- VariantPhysicalType pt = GetPhysicalType();
- DCHECK(pt == VariantPhysicalType::BOOLEAN_TRUE
- || pt == VariantPhysicalType::BOOLEAN_FALSE);
- return pt == VariantPhysicalType::BOOLEAN_TRUE;
+bool VariantValue::GetBoolean(bool* out) const {
+ VariantPhysicalType pt;
+ if (!AsPrimitive(&pt)) return false;
+ if (pt == VariantPhysicalType::BOOLEAN_TRUE) { *out = true; return true; }
+ if (pt == VariantPhysicalType::BOOLEAN_FALSE) { *out = false; return true; }
+ return false;
}
-int8_t VariantValue::GetInt8() const {
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::INT8);
- return static_cast<int8_t>(data_[1]);
+bool VariantValue::GetInt8(int8_t* out) const {
+ return ReadValueOfType(VariantPhysicalType::INT8, out);
}
-int16_t VariantValue::GetInt16() const {
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::INT16);
- return ReadValue<int16_t>();
+bool VariantValue::GetInt16(int16_t* out) const {
+ return ReadValueOfType(VariantPhysicalType::INT16, out);
}
-int32_t VariantValue::GetInt32() const {
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::INT32);
- return ReadValue<int32_t>();
+bool VariantValue::GetInt32(int32_t* out) const {
+ return ReadValueOfType(VariantPhysicalType::INT32, out);
}
-int64_t VariantValue::GetInt64() const {
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::INT64);
- return ReadValue<int64_t>();
+bool VariantValue::GetInt64(int64_t* out) const {
+ return ReadValueOfType(VariantPhysicalType::INT64, out);
}
-float VariantValue::GetFloat() const {
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::FLOAT);
- return ReadValue<float>();
+bool VariantValue::GetFloat(float* out) const {
+ return ReadValueOfType(VariantPhysicalType::FLOAT, out);
}
-double VariantValue::GetDouble() const {
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::DOUBLE);
- return ReadValue<double>();
+bool VariantValue::GetDouble(double* out) const {
+ return ReadValueOfType(VariantPhysicalType::DOUBLE, out);
}
-StringValue VariantValue::GetString() const {
+bool VariantValue::GetString(StringValue* out) const {
+ if (data_ == nullptr || len_ < 1) return false;
VariantBasicType bt = GetBasicType();
if (bt == VariantBasicType::SHORT_STRING) {
- int str_len = (data_[0] >> 2) & 0x3F;
- StringValue sv;
- sv.Assign(reinterpret_cast<char*>(const_cast<uint8_t*>(data_ + 1)),
- str_len);
- return sv;
+ uint32_t str_len = (data_[0] >> 2) & 0x3F;
+ if (str_len > len_ - 1) return false;
+ out->Assign(reinterpret_cast<char*>(const_cast<uint8_t*>(data_ + 1)), str_len);
+ return true;
+ }
+ if (bt != VariantBasicType::PRIMITIVE
+ || GetPhysicalType() != VariantPhysicalType::STRING || len_ < 5) {
+ return false;
}
- DCHECK_EQ(bt, VariantBasicType::PRIMITIVE);
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::STRING);
uint32_t str_len = ReadUint(data_ + 1, 4);
- StringValue sv;
- sv.Assign(reinterpret_cast<char*>(const_cast<uint8_t*>(data_ + 5)),
- str_len);
- return sv;
+ if (str_len > len_ - 5) return false;
+ out->Assign(reinterpret_cast<char*>(const_cast<uint8_t*>(data_ + 5)), str_len);
+ return true;
}
-StringValue VariantValue::GetBinary() const {
- DCHECK_EQ(GetBasicType(), VariantBasicType::PRIMITIVE);
- DCHECK_EQ(GetPhysicalType(), VariantPhysicalType::BINARY);
+bool VariantValue::GetBinary(StringValue* out) const {
+ VariantPhysicalType pt;
+ if (!AsPrimitive(&pt) || pt != VariantPhysicalType::BINARY || len_ < 5) return false;
uint32_t bin_len = ReadUint(data_ + 1, 4);
- StringValue sv;
- sv.Assign(reinterpret_cast<char*>(const_cast<uint8_t*>(data_ + 5)),
- bin_len);
- return sv;
+ if (bin_len > len_ - 5) return false;
+ out->Assign(reinterpret_cast<char*>(const_cast<uint8_t*>(data_ + 5)), bin_len);
+ return true;
}
// --- Object accessors ---
@@ -240,88 +261,85 @@ uint32_t VariantValue::ObjectOffsetSize() const {
return ((data_[0] >> 2) & 0x03) + 1;
}
-uint32_t VariantValue::ObjectNumFields() const {
- DCHECK_EQ(GetBasicType(), VariantBasicType::OBJECT);
- bool is_large = (data_[0] >> 6) & 0x01;
- uint32_t size_bytes = is_large ? 4 : 1;
- return ReadUint(data_ + 1, size_bytes);
-}
-
-const uint8_t* VariantValue::ObjectFieldIdsStart() const {
+bool VariantValue::ParseObjectLayout(ObjectLayout* out) const {
+ if (data_ == nullptr || len_ < 1) return false;
+ if (GetBasicType() != VariantBasicType::OBJECT) return false;
bool is_large = (data_[0] >> 6) & 0x01;
- uint32_t size_bytes = is_large ? 4 : 1;
- return data_ + 1 + size_bytes;
-}
-
-const uint8_t* VariantValue::ObjectOffsetsStart() const {
- uint32_t num_fields = ObjectNumFields();
+ uint32_t num_size = is_large ? 4 : 1;
+ if (len_ < 1 + num_size) return false;
+ uint32_t num_fields = ReadUint(data_ + 1, num_size);
uint32_t field_id_size = ObjectFieldIdSize();
- return ObjectFieldIdsStart() + num_fields * field_id_size;
-}
-
-const uint8_t* VariantValue::ObjectDataStart() const {
- uint32_t num_fields = ObjectNumFields();
uint32_t offset_size = ObjectOffsetSize();
- return ObjectOffsetsStart() + (num_fields + 1) * offset_size;
+ // Layout: header | field_ids[num_fields] | offsets[num_fields + 1] | data.
+ // 64-bit math so a corrupt (large) num_fields cannot overflow the bound check.
+ uint64_t data_start_off = static_cast<uint64_t>(1) + num_size
+ + static_cast<uint64_t>(num_fields) * field_id_size
+ + (static_cast<uint64_t>(num_fields) + 1) * offset_size;
+ if (data_start_off > len_) return false;
+ out->num_fields = num_fields;
+ out->field_ids = data_ + 1 + num_size;
+ out->offsets = out->field_ids + static_cast<uint64_t>(num_fields) * field_id_size;
+ out->data = data_ + data_start_off;
+ out->data_len = static_cast<uint32_t>(len_ - data_start_off);
+ return true;
}
-uint32_t VariantValue::GetObjectSize() const {
- if (GetBasicType() != VariantBasicType::OBJECT) return 0;
- return ObjectNumFields();
+bool VariantValue::GetObjectSize(uint32_t* out) const {
+ ObjectLayout layout;
+ if (!ParseObjectLayout(&layout)) return false;
+ *out = layout.num_fields;
+ return true;
}
-bool VariantValue::GetFieldByName(string_view name,
- VariantValue* result) const {
- if (GetBasicType() != VariantBasicType::OBJECT) return false;
+bool VariantValue::GetFieldByName(string_view name, VariantValue* result) const {
if (metadata_ == nullptr) return false;
+ ObjectLayout layout;
+ if (!ParseObjectLayout(&layout)) return false;
int field_id = metadata_->FindFieldId(name);
if (field_id < 0) return false;
- uint32_t num_fields = ObjectNumFields();
- uint32_t field_id_size = ObjectFieldIdSize();
- const uint8_t* field_ids = ObjectFieldIdsStart();
-
// Search for this field_id in the object's field_id array.
- int field_index = -1;
- for (uint32_t i = 0; i < num_fields; ++i) {
- uint32_t fid = ReadUint(field_ids + i * field_id_size, field_id_size);
- if (fid == field_id) {
- field_index = i;
- break;
- }
+ uint32_t field_id_size = ObjectFieldIdSize();
+ for (uint32_t i = 0; i < layout.num_fields; ++i) {
+ uint32_t fid = ReadUint(layout.field_ids + i * field_id_size, field_id_size);
+ if (fid == static_cast<uint32_t>(field_id)) return FieldFromLayout(layout, i, result);
}
- if (field_index < 0) return false;
-
- return GetFieldByIndex(field_index, result);
+ return false;
}
-bool VariantValue::GetFieldByIndex(uint32_t index, VariantValue* result) const {
- if (GetBasicType() != VariantBasicType::OBJECT) return false;
- uint32_t num_fields = ObjectNumFields();
- if (index >= num_fields) return false;
-
+bool VariantValue::FieldFromLayout(const ObjectLayout& layout, uint32_t index,
+ VariantValue* result) const {
+ if (index >= layout.num_fields) return false;
uint32_t offset_size = ObjectOffsetSize();
- const uint8_t* offsets = ObjectOffsetsStart();
- const uint8_t* data_start = ObjectDataStart();
+ uint32_t start = ReadUint(layout.offsets + index * offset_size, offset_size);
+ uint32_t end = ReadUint(layout.offsets + (index + 1) * offset_size, offset_size);
+ if (end < start || end > layout.data_len) return false;
+ *result = VariantValue(layout.data + start, end - start, metadata_);
+ return true;
+}
- uint32_t field_offset =
- ReadUint(offsets + index * offset_size, offset_size);
- uint32_t next_offset =
- ReadUint(offsets + (index + 1) * offset_size, offset_size);
- uint32_t field_len = next_offset - field_offset;
+bool VariantValue::GetFieldByIndex(uint32_t index, VariantValue* result) const {
+ ObjectLayout layout;
+ if (!ParseObjectLayout(&layout)) return false;
+ return FieldFromLayout(layout, index, result);
+}
- *result = VariantValue(data_start + field_offset, field_len, metadata_);
+bool VariantValue::FieldNameFromLayout(const ObjectLayout& layout, uint32_t index,
+ string_view* out) const {
+ if (index >= layout.num_fields) return false;
+ uint32_t field_id_size = ObjectFieldIdSize();
+ uint32_t fid = ReadUint(layout.field_ids + index * field_id_size, field_id_size);
+ uint32_t dict_size = (metadata_ != nullptr) ? metadata_->DictionarySize() : 0;
+ if (fid >= dict_size) return false;
+ *out = metadata_->GetFieldName(fid);
return true;
}
-string_view VariantValue::GetFieldNameByIndex(uint32_t index) const {
- DCHECK_EQ(GetBasicType(), VariantBasicType::OBJECT);
- uint32_t field_id_size = ObjectFieldIdSize();
- const uint8_t* field_ids = ObjectFieldIdsStart();
- uint32_t fid =
- ReadUint(field_ids + index * field_id_size, field_id_size);
- return metadata_->GetFieldName(fid);
+bool VariantValue::GetFieldNameByIndex(uint32_t index, string_view* out) const {
+ ObjectLayout layout;
+ if (!ParseObjectLayout(&layout)) return false;
+ return FieldNameFromLayout(layout, index, out);
}
// --- Array accessors ---
@@ -331,47 +349,47 @@ uint32_t VariantValue::ArrayOffsetSize() const {
return ((data_[0] >> 2) & 0x03) + 1;
}
-uint32_t VariantValue::ArrayNumElements() const {
- DCHECK_EQ(GetBasicType(), VariantBasicType::ARRAY);
+bool VariantValue::ParseArrayLayout(ArrayLayout* out) const {
+ if (data_ == nullptr || len_ < 1) return false;
+ if (GetBasicType() != VariantBasicType::ARRAY) return false;
bool is_large = (data_[0] >> 4) & 0x01;
- uint32_t size_bytes = is_large ? 4 : 1;
- return ReadUint(data_ + 1, size_bytes);
+ uint32_t num_size = is_large ? 4 : 1;
+ if (len_ < 1 + num_size) return false;
+ uint32_t num_elems = ReadUint(data_ + 1, num_size);
+ uint32_t offset_size = ArrayOffsetSize();
+ // Layout: header | offsets[num_elems + 1] | data.
+ uint64_t data_start_off = static_cast<uint64_t>(1) + num_size
+ + (static_cast<uint64_t>(num_elems) + 1) * offset_size;
+ if (data_start_off > len_) return false;
+ out->num_elems = num_elems;
+ out->offsets = data_ + 1 + num_size;
+ out->data = data_ + data_start_off;
+ out->data_len = static_cast<uint32_t>(len_ - data_start_off);
+ return true;
}
-const uint8_t* VariantValue::ArrayOffsetsStart() const {
- bool is_large = (data_[0] >> 4) & 0x01;
- uint32_t size_bytes = is_large ? 4 : 1;
- return data_ + 1 + size_bytes;
+bool VariantValue::GetArraySize(uint32_t* out) const {
+ ArrayLayout layout;
+ if (!ParseArrayLayout(&layout)) return false;
+ *out = layout.num_elems;
+ return true;
}
-const uint8_t* VariantValue::ArrayDataStart() const {
- uint32_t num_elements = ArrayNumElements();
+bool VariantValue::ElementFromLayout(const ArrayLayout& layout, uint32_t index,
+ VariantValue* result) const {
+ if (index >= layout.num_elems) return false;
uint32_t offset_size = ArrayOffsetSize();
- return ArrayOffsetsStart() + (num_elements + 1) * offset_size;
-}
-
-uint32_t VariantValue::GetArraySize() const {
- if (GetBasicType() != VariantBasicType::ARRAY) return 0;
- return ArrayNumElements();
+ uint32_t start = ReadUint(layout.offsets + index * offset_size, offset_size);
+ uint32_t end = ReadUint(layout.offsets + (index + 1) * offset_size, offset_size);
+ if (end < start || end > layout.data_len) return false;
+ *result = VariantValue(layout.data + start, end - start, metadata_);
+ return true;
}
bool VariantValue::GetArrayElement(uint32_t index, VariantValue* result) const {
- if (GetBasicType() != VariantBasicType::ARRAY) return false;
- uint32_t num_elements = ArrayNumElements();
- if (index >= num_elements) return false;
-
- uint32_t offset_size = ArrayOffsetSize();
- const uint8_t* offsets = ArrayOffsetsStart();
- const uint8_t* data_start = ArrayDataStart();
-
- uint32_t elem_offset =
- ReadUint(offsets + index * offset_size, offset_size);
- uint32_t next_offset =
- ReadUint(offsets + (index + 1) * offset_size, offset_size);
- uint32_t elem_len = next_offset - elem_offset;
-
- *result = VariantValue(data_start + elem_offset, elem_len, metadata_);
- return true;
+ ArrayLayout layout;
+ if (!ParseArrayLayout(&layout)) return false;
+ return ElementFromLayout(layout, index, result);
}
// --- Path navigation ---
@@ -429,12 +447,35 @@ bool VariantValue::NavigatePath(const string& path,
using JsonWriter = rapidjson::Writer<rapidjson::StringBuffer>;
-static Status ValueToJson(const VariantValue& val,
- const VariantMetadata& metadata, JsonWriter* writer,
- impala_udf::FunctionContext* ctx = nullptr) {
+// Builds the error status returned when a variant value cannot be decoded because it is
+// truncated or its payload would read out of bounds. 'what' names the offending part,
+// e.g. "INT32" or "object field name".
+static Status MalformedVariant(const char* what) {
+ return Status(Substitute("Malformed variant $0", what));
+}
+
+// Serializes a VariantValue to JSON. A friend of VariantValue (declared in the header) so
+// it can parse each object/array layout once via the private layout helpers and reuse it
+// across the field/element loop, while keeping rapidjson out of variant-value.h.
+struct VariantJsonSerializer {
+ static Status Write(const VariantValue& val, const VariantMetadata& metadata,
+ JsonWriter* writer, int depth = ColumnType::MAX_NESTING_DEPTH);
+};
+
+Status VariantJsonSerializer::Write(const VariantValue& val,
+ const VariantMetadata& metadata, JsonWriter* writer, int depth) {
+ if (UNLIKELY(depth <= 0)) {
+ return Status("Variant value nesting exceeds the maximum allowed depth");
+ }
+ if (UNLIKELY(!val.IsValid() || val.Len() < 1)) {
+ return MalformedVariant("value: empty or truncated value buffer");
+ }
switch (val.GetBasicType()) {
case VariantBasicType::SHORT_STRING: {
- StringValue sv = val.GetString();
+ StringValue sv;
+ if (UNLIKELY(!val.GetString(&sv))) {
+ return MalformedVariant("short string");
+ }
writer->String(sv.Ptr(), sv.Len());
return Status::OK();
}
@@ -449,110 +490,136 @@ static Status ValueToJson(const VariantValue& val,
case VariantPhysicalType::BOOLEAN_FALSE:
writer->Bool(false);
break;
- case VariantPhysicalType::INT8:
- writer->Int(val.GetInt8());
+ case VariantPhysicalType::INT8: {
+ int8_t v;
+ if (UNLIKELY(!val.GetInt8(&v))) return MalformedVariant("INT8");
+ writer->Int(v);
break;
- case VariantPhysicalType::INT16:
- writer->Int(val.GetInt16());
+ }
+ case VariantPhysicalType::INT16: {
+ int16_t v;
+ if (UNLIKELY(!val.GetInt16(&v))) return MalformedVariant("INT16");
+ writer->Int(v);
break;
- case VariantPhysicalType::INT32:
- writer->Int(val.GetInt32());
+ }
+ case VariantPhysicalType::INT32: {
+ int32_t v;
+ if (UNLIKELY(!val.GetInt32(&v))) return MalformedVariant("INT32");
+ writer->Int(v);
break;
- case VariantPhysicalType::INT64:
- writer->Int64(val.GetInt64());
+ }
+ case VariantPhysicalType::INT64: {
+ int64_t v;
+ if (UNLIKELY(!val.GetInt64(&v))) return MalformedVariant("INT64");
+ writer->Int64(v);
break;
+ }
case VariantPhysicalType::FLOAT: {
+ float v;
+ if (UNLIKELY(!val.GetFloat(&v))) return MalformedVariant("FLOAT");
char buf[24];
- int n = snprintf(buf, sizeof(buf), "%g", val.GetFloat());
+ int n = snprintf(buf, sizeof(buf), "%g", v);
writer->RawValue(buf, n, rapidjson::kNumberType);
break;
}
- case VariantPhysicalType::DOUBLE:
- writer->Double(val.GetDouble());
+ case VariantPhysicalType::DOUBLE: {
+ double v;
+ if (UNLIKELY(!val.GetDouble(&v))) return MalformedVariant("DOUBLE");
+ writer->Double(v);
break;
+ }
case VariantPhysicalType::STRING: {
- StringValue sv = val.GetString();
+ StringValue sv;
+ if (UNLIKELY(!val.GetString(&sv))) return MalformedVariant("STRING");
writer->String(sv.Ptr(), sv.Len());
break;
}
case VariantPhysicalType::DATE: {
- DateValue dv(static_cast<int64_t>(val.ReadValue<int32_t>()));
+ int32_t days;
+ if (UNLIKELY(!val.ReadValue(&days))) return MalformedVariant("DATE");
+ DateValue dv(static_cast<int64_t>(days));
char buf[SimpleDateFormatTokenizer::DEFAULT_DATE_FMT_LEN];
int n = DateParser::FormatDefault(dv, buf);
- if (LIKELY(n > 0)) {
- DCHECK_LE(n, sizeof(buf));
- writer->String(buf, n);
- } else {
- if (ctx) {
- ctx->AddWarning("Invalid DATE value in VARIANT");
- }
- writer->String("<invalid-date>");
- }
+ if (UNLIKELY(n <= 0)) return Status("Variant DATE value out of range");
+ DCHECK_LE(n, sizeof(buf));
+ writer->String(buf, n);
break;
}
case VariantPhysicalType::DECIMAL4: {
+ int32_t unscaled;
+ if (UNLIKELY(!val.ReadValue(&unscaled, 2))) {
+ return MalformedVariant("DECIMAL4");
+ }
int scale = val.Data()[1];
- int32_t unscaled = val.ReadValue<int32_t>(2);
- string s = Decimal4Value(unscaled).ToString(9, scale);
+ if (UNLIKELY(scale > ColumnType::MAX_DECIMAL4_PRECISION)) {
+ return MalformedVariant("DECIMAL4: scale out of range");
+ }
+ string s = Decimal4Value(unscaled).ToString(
+ ColumnType::MAX_DECIMAL4_PRECISION, scale);
writer->RawValue(s.data(), s.size(), rapidjson::kNumberType);
break;
}
case VariantPhysicalType::DECIMAL8: {
+ int64_t unscaled;
+ if (UNLIKELY(!val.ReadValue(&unscaled, 2))) {
+ return MalformedVariant("DECIMAL8");
+ }
int scale = val.Data()[1];
- int64_t unscaled = val.ReadValue<int64_t>(2);
- string s = Decimal8Value(unscaled).ToString(18, scale);
+ if (UNLIKELY(scale > ColumnType::MAX_DECIMAL8_PRECISION)) {
+ return MalformedVariant("DECIMAL8: scale out of range");
+ }
+ string s = Decimal8Value(unscaled).ToString(
+ ColumnType::MAX_DECIMAL8_PRECISION, scale);
writer->RawValue(s.data(), s.size(), rapidjson::kNumberType);
break;
}
case VariantPhysicalType::DECIMAL16: {
+ __int128_t unscaled;
+ if (UNLIKELY(!val.ReadValue(&unscaled, 2))) {
+ return MalformedVariant("DECIMAL16");
+ }
int scale = val.Data()[1];
- __int128_t unscaled = val.ReadValue<__int128_t>(2);
- string s = Decimal16Value(unscaled).ToString(38, scale);
+ if (UNLIKELY(scale > ColumnType::MAX_PRECISION)) {
+ return MalformedVariant("DECIMAL16: scale out of range");
+ }
+ string s = Decimal16Value(unscaled).ToString(
+ ColumnType::MAX_PRECISION, scale);
writer->RawValue(s.data(), s.size(), rapidjson::kNumberType);
break;
}
case VariantPhysicalType::TIMESTAMPNTZ: {
- int64_t micros = val.ReadValue<int64_t>();
+ int64_t micros;
+ if (UNLIKELY(!val.ReadValue(µs))) {
+ return MalformedVariant("TIMESTAMP");
+ }
TimestampValue ts = TimestampValue::UtcFromUnixTimeMicros(micros);
char buf[SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_FMT_LEN];
int n = TimestampParser::FormatDefault(ts.date(), ts.time(), buf);
- if (LIKELY(n > 0)) {
- DCHECK_LE(n, sizeof(buf));
- writer->String(buf, n);
- } else {
- if (ctx) {
- ctx->AddWarning("Invalid TIMESTAMP value in VARIANT");
- }
- writer->String("<invalid-timestamp>");
- }
+ if (UNLIKELY(n <= 0)) return Status("Variant TIMESTAMP value out of range");
+ DCHECK_LE(n, sizeof(buf));
+ writer->String(buf, n);
break;
}
case VariantPhysicalType::TIMESTAMPNTZ_NANOS: {
- int64_t nanos = val.ReadValue<int64_t>();
+ int64_t nanos;
+ if (UNLIKELY(!val.ReadValue(&nanos))) {
+ return MalformedVariant("TIMESTAMP");
+ }
TimestampValue ts =
TimestampValue::UtcFromUnixTimeLimitedRangeNanos(nanos);
char buf[SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_FMT_LEN];
int n = TimestampParser::FormatDefault(ts.date(), ts.time(), buf);
- if (LIKELY(n > 0)) {
- DCHECK_LE(n, sizeof(buf));
- writer->String(buf, n);
- } else {
- if (ctx) {
- ctx->AddWarning("Invalid timestamp value in VARIANT");
- }
- writer->String("<invalid-timestamp>");
- }
+ if (UNLIKELY(n <= 0)) return Status("Variant TIMESTAMP value out of range");
+ DCHECK_LE(n, sizeof(buf));
+ writer->String(buf, n);
break;
}
case VariantPhysicalType::BINARY: {
- StringValue sv = val.GetBinary();
+ StringValue sv;
+ if (UNLIKELY(!val.GetBinary(&sv))) return MalformedVariant("BINARY");
int64_t out_max;
if (UNLIKELY(!Base64EncodeBufLen(sv.Len(), &out_max))) {
- if (ctx) {
- ctx->AddWarning("Invalid BINARY value in VARIANT");
- }
- writer->String("<invalid-binary>");
- break;
+ return Status("Variant BINARY value too large to encode");
}
string encoded(out_max, '\0');
unsigned out_len;
@@ -571,29 +638,40 @@ static Status ValueToJson(const VariantValue& val,
return Status::OK();
}
case VariantBasicType::OBJECT: {
+ // Parse and bounds-check the header + offset table once, then reuse the layout for
+ // every field rather than re-parsing on each accessor call.
+ VariantValue::ObjectLayout layout;
+ if (UNLIKELY(!val.ParseObjectLayout(&layout))) {
+ return MalformedVariant("object");
+ }
writer->StartObject();
- uint32_t num_fields = val.GetObjectSize();
- for (uint32_t i = 0; i < num_fields; ++i) {
- string_view field_name = val.GetFieldNameByIndex(i);
+ for (uint32_t i = 0; i < layout.num_fields; ++i) {
+ string_view field_name;
+ if (UNLIKELY(!val.FieldNameFromLayout(layout, i, &field_name))) {
+ return MalformedVariant("object field name");
+ }
writer->Key(field_name.data(), field_name.size());
VariantValue child;
- if (!val.GetFieldByIndex(i, &child)) {
+ if (UNLIKELY(!val.FieldFromLayout(layout, i, &child))) {
return Status("Failed to read object field");
}
- RETURN_IF_ERROR(ValueToJson(child, metadata, writer));
+ RETURN_IF_ERROR(Write(child, metadata, writer, depth - 1));
}
writer->EndObject();
return Status::OK();
}
case VariantBasicType::ARRAY: {
+ VariantValue::ArrayLayout layout;
+ if (UNLIKELY(!val.ParseArrayLayout(&layout))) {
+ return MalformedVariant("array");
+ }
writer->StartArray();
- uint32_t num_elements = val.GetArraySize();
- for (uint32_t i = 0; i < num_elements; ++i) {
+ for (uint32_t i = 0; i < layout.num_elems; ++i) {
VariantValue elem;
- if (!val.GetArrayElement(i, &elem)) {
+ if (UNLIKELY(!val.ElementFromLayout(layout, i, &elem))) {
return Status("Failed to read array element");
}
- RETURN_IF_ERROR(ValueToJson(elem, metadata, writer));
+ RETURN_IF_ERROR(Write(elem, metadata, writer, depth - 1));
}
writer->EndArray();
return Status::OK();
@@ -612,7 +690,8 @@ Status VariantValue::ToJson(std::string* json_out) const {
DCHECK(metadata_ != nullptr);
auto buffer = CreateStringBuffer(Len() * 2);
JsonWriter writer(buffer);
- RETURN_IF_ERROR(ValueToJson(*this, *metadata_, &writer));
+ RETURN_IF_ERROR(VariantJsonSerializer::Write(
+ *this, *metadata_, &writer));
json_out->assign(buffer.GetString(), buffer.GetSize());
return Status::OK();
}
@@ -622,7 +701,8 @@ Status VariantValue::ToJson(impala_udf::FunctionContext* ctx,
DCHECK(metadata_ != nullptr);
auto buffer = CreateStringBuffer(Len() * 2);
JsonWriter writer(buffer);
- RETURN_IF_ERROR(ValueToJson(*this, *metadata_, &writer, ctx));
+ RETURN_IF_ERROR(VariantJsonSerializer::Write(
+ *this, *metadata_, &writer));
*result = impala_udf::StringVal::CopyFrom(ctx,
reinterpret_cast<const uint8_t*>(buffer.GetString()), buffer.GetSize());
return Status::OK();
@@ -634,7 +714,8 @@ Status VariantValue::ToJson(std::ostream* out) const {
// some bytes have already been written to 'writer') does not leak partial JSON.
auto buffer = CreateStringBuffer(Len() * 2);
JsonWriter writer(buffer);
- RETURN_IF_ERROR(ValueToJson(*this, *metadata_, &writer));
+ RETURN_IF_ERROR(VariantJsonSerializer::Write(
+ *this, *metadata_, &writer));
out->write(buffer.GetString(), buffer.GetSize());
return Status::OK();
}
diff --git a/be/src/runtime/variant-value.h b/be/src/runtime/variant-value.h
index 91963b45e..249d6f38c 100644
--- a/be/src/runtime/variant-value.h
+++ b/be/src/runtime/variant-value.h
@@ -134,6 +134,7 @@ class VariantMetadata {
const uint8_t* offsets_ = nullptr;
const uint8_t* string_data_ = nullptr;
+ uint32_t string_data_len_ = 0; // number of bytes in the string-data region
uint32_t dict_size_ = 0;
uint8_t version_ = 0;
uint8_t offset_size_ = 0; // 1, 2, 3, or 4 bytes per offset
@@ -154,7 +155,7 @@ class VariantMetadata {
class VariantValue {
public:
VariantValue() = default;
- VariantValue(const uint8_t* data, int32_t len, const VariantMetadata* metadata)
+ VariantValue(const uint8_t* data, uint32_t len, const VariantMetadata* metadata)
: data_(data), len_(len), metadata_(metadata) {}
// Returns the basic type of this value.
@@ -166,33 +167,36 @@ class VariantValue {
// Returns true if this value is null.
bool IsNull() const;
- // Scalar accessors. Caller must ensure the physical type matches.
- bool GetBoolean() const;
- int8_t GetInt8() const;
- int16_t GetInt16() const;
- int32_t GetInt32() const;
- int64_t GetInt64() const;
- float GetFloat() const;
- double GetDouble() const;
- StringValue GetString() const;
- StringValue GetBinary() const;
-
- // Object access.
- uint32_t GetObjectSize() const;
- // Gets the field value by field name. Returns false if field not found.
- bool GetFieldByName(std::string_view name, VariantValue* result) const;
+ // Scalar accessors. Each returns false and leaves '*out' unchanged if the value is not
+ // of the expected physical type or its payload would read out of bounds.
+ [[nodiscard]] bool GetBoolean(bool* out) const;
+ [[nodiscard]] bool GetInt8(int8_t* out) const;
+ [[nodiscard]] bool GetInt16(int16_t* out) const;
+ [[nodiscard]] bool GetInt32(int32_t* out) const;
+ [[nodiscard]] bool GetInt64(int64_t* out) const;
+ [[nodiscard]] bool GetFloat(float* out) const;
+ [[nodiscard]] bool GetDouble(double* out) const;
+ [[nodiscard]] bool GetString(StringValue* out) const;
+ [[nodiscard]] bool GetBinary(StringValue* out) const;
+
+ // Object access. Each returns false if this value is not a well-formed object (a
+ // corrupt or truncated header/offset table), if 'index' is out of range, or if the
+ // field's encoding is out of bounds.
+ [[nodiscard]] bool GetObjectSize(uint32_t* out) const;
+ // Gets the field value by field name. Returns false if the field is not found.
+ [[nodiscard]] bool GetFieldByName(std::string_view name, VariantValue* result) const;
// Gets the field value by position index.
- bool GetFieldByIndex(uint32_t index, VariantValue* result) const;
+ [[nodiscard]] bool GetFieldByIndex(uint32_t index, VariantValue* result) const;
// Gets the field name at position index in this object.
- std::string_view GetFieldNameByIndex(uint32_t index) const;
+ [[nodiscard]] bool GetFieldNameByIndex(uint32_t index, std::string_view* out) const;
- // Array access.
- uint32_t GetArraySize() const;
- bool GetArrayElement(uint32_t index, VariantValue* result) const;
+ // Array access. Returns false on a corrupt/truncated array or an out-of-range index.
+ [[nodiscard]] bool GetArraySize(uint32_t* out) const;
+ [[nodiscard]] bool GetArrayElement(uint32_t index, VariantValue* result) const;
// Navigate a dotted path like "field.nested[0].value".
// Returns false if the path cannot be resolved.
- bool NavigatePath(const std::string& path, VariantValue* result) const;
+ [[nodiscard]] bool NavigatePath(const std::string& path, VariantValue* result) const;
// Serialize this variant value to JSON string.
Status ToJson(std::string* json_out) const;
@@ -205,33 +209,78 @@ class VariantValue {
const uint8_t* Data() const { return data_; }
uint32_t Len() const { return len_; }
- // Reads a primitive value of type T from the payload (data_ + offset).
- // Default offset is 1 (immediately after the header byte).
+ // Reads a primitive of type T from the payload at data_ + offset (offset defaults to 1,
+ // just past the header byte). Returns false without modifying '*out' if the read would
+ // extend past the end of the value buffer.
template <typename T>
- T ReadValue(uint32_t offset = 1) const {
- DCHECK_GE(len_, offset + sizeof(T));
- T val;
- memcpy(&val, data_ + offset, sizeof(T));
- return val;
+ [[nodiscard]] bool ReadValue(T* out, uint32_t offset = 1) const {
+ if (offset > len_ || sizeof(T) > len_ - offset) return false;
+ memcpy(out, data_ + offset, sizeof(T));
+ return true;
}
private:
// Helper to read a variable-width unsigned integer.
static uint32_t ReadUint(const uint8_t* data, uint32_t size);
- // For OBJECT: parse the header to get field count and internal layout.
+ // If this value is a PRIMITIVE with a readable header byte, sets '*pt' to its physical
+ // type and returns true; otherwise returns false.
+ [[nodiscard]] bool AsPrimitive(VariantPhysicalType* pt) const {
+ if (data_ == nullptr || len_ < 1) return false;
+ if (GetBasicType() != VariantBasicType::PRIMITIVE) return false;
+ *pt = GetPhysicalType();
+ return true;
+ }
+
+ // Checks this value is a PRIMITIVE of 'expected' physical type, then reads a T-sized
+ // payload just past the header byte (see ReadValue()). Returns false without modifying
+ // '*out' on a type mismatch or a truncated payload. Shared by the typed scalar getters.
+ template <typename T>
+ [[nodiscard]] bool ReadValueOfType(VariantPhysicalType expected, T* out) const {
+ VariantPhysicalType pt;
+ if (!AsPrimitive(&pt) || pt != expected) return false;
+ return ReadValue(out);
+ }
+
+ // Validated internal layout of an object/array value. The pointers and lengths are
+ // guaranteed to lie within the value buffer once the matching Parse*() returns true.
+ struct ObjectLayout {
+ uint32_t num_fields;
+ const uint8_t* field_ids;
+ const uint8_t* offsets;
+ const uint8_t* data;
+ uint32_t data_len;
+ };
+ struct ArrayLayout {
+ uint32_t num_elems;
+ const uint8_t* offsets;
+ const uint8_t* data;
+ uint32_t data_len;
+ };
+ // Parse and bounds-check the object/array header + offset table. Returns false (without
+ // reading out of bounds) if the value is not that type or the header/tables do not fit.
+ [[nodiscard]] bool ParseObjectLayout(ObjectLayout* out) const;
+ [[nodiscard]] bool ParseArrayLayout(ArrayLayout* out) const;
+
+ // Extract a single field/element from an already-parsed, validated layout, bounds-
+ // checking the individual entry's offsets (and, for a field name, that its dictionary
+ // id is in range). Shared by the public single-access accessors and by JSON
+ // serialization, which parses the layout once and reuses it across the whole loop.
+ [[nodiscard]] bool FieldFromLayout(const ObjectLayout& layout, uint32_t index,
+ VariantValue* result) const;
+ [[nodiscard]] bool FieldNameFromLayout(const ObjectLayout& layout, uint32_t index,
+ std::string_view* out) const;
+ [[nodiscard]] bool ElementFromLayout(const ArrayLayout& layout, uint32_t index,
+ VariantValue* result) const;
+
+ // JSON serialization lives in the .cc (so rapidjson stays out of this header) but needs
+ // the private layout helpers above to parse each object/array once and reuse it.
+ friend struct VariantJsonSerializer;
+
+ // Bit-width helpers derived from the header byte (require a readable header).
uint32_t ObjectFieldIdSize() const;
uint32_t ObjectOffsetSize() const;
- uint32_t ObjectNumFields() const;
- const uint8_t* ObjectFieldIdsStart() const;
- const uint8_t* ObjectOffsetsStart() const;
- const uint8_t* ObjectDataStart() const;
-
- // For ARRAY: parse the header to get element count and offsets.
uint32_t ArrayOffsetSize() const;
- uint32_t ArrayNumElements() const;
- const uint8_t* ArrayOffsetsStart() const;
- const uint8_t* ArrayDataStart() const;
const uint8_t* data_ = nullptr;
uint32_t len_ = 0;
diff --git a/be/src/util/variant-util-test.cc b/be/src/util/variant-util-test.cc
index bf78a7bc7..7d604a29c 100644
--- a/be/src/util/variant-util-test.cc
+++ b/be/src/util/variant-util-test.cc
@@ -176,11 +176,14 @@ TEST(VariantUtilTest, BooleanValue) {
vector<uint8_t> true_bytes = BuildBoolean(true);
VariantValue true_val(true_bytes.data(), true_bytes.size(), &metadata);
EXPECT_FALSE(true_val.IsNull());
- EXPECT_TRUE(true_val.GetBoolean());
+ bool b = false;
+ EXPECT_TRUE(true_val.GetBoolean(&b));
+ EXPECT_TRUE(b);
vector<uint8_t> false_bytes = BuildBoolean(false);
VariantValue false_val(false_bytes.data(), false_bytes.size(), &metadata);
- EXPECT_FALSE(false_val.GetBoolean());
+ EXPECT_TRUE(false_val.GetBoolean(&b));
+ EXPECT_FALSE(b);
}
TEST(VariantUtilTest, Int32Value) {
@@ -192,7 +195,9 @@ TEST(VariantUtilTest, Int32Value) {
VariantValue val(val_bytes.data(), val_bytes.size(), &metadata);
EXPECT_EQ(val.GetBasicType(), VariantBasicType::PRIMITIVE);
EXPECT_EQ(val.GetPhysicalType(), VariantPhysicalType::INT32);
- EXPECT_EQ(42, val.GetInt32());
+ int32_t int_val;
+ EXPECT_TRUE(val.GetInt32(&int_val));
+ EXPECT_EQ(42, int_val);
}
TEST(VariantUtilTest, ShortString) {
@@ -203,7 +208,8 @@ TEST(VariantUtilTest, ShortString) {
vector<uint8_t> val_bytes = BuildShortString("hello");
VariantValue val(val_bytes.data(), val_bytes.size(), &metadata);
EXPECT_EQ(val.GetBasicType(), VariantBasicType::SHORT_STRING);
- StringValue sv = val.GetString();
+ StringValue sv;
+ ASSERT_TRUE(val.GetString(&sv));
EXPECT_EQ(string(sv.Ptr(), sv.Len()), "hello");
}
@@ -220,14 +226,19 @@ TEST(VariantUtilTest, SimpleObject) {
VariantValue obj(obj_bytes.data(), obj_bytes.size(), &metadata);
EXPECT_EQ(obj.GetBasicType(), VariantBasicType::OBJECT);
- EXPECT_EQ(2, obj.GetObjectSize());
+ uint32_t obj_size;
+ EXPECT_TRUE(obj.GetObjectSize(&obj_size));
+ EXPECT_EQ(2, obj_size);
VariantValue field_val;
+ int32_t field_int;
EXPECT_TRUE(obj.GetFieldByName("age", &field_val));
- EXPECT_EQ(30, field_val.GetInt32());
+ EXPECT_TRUE(field_val.GetInt32(&field_int));
+ EXPECT_EQ(30, field_int);
EXPECT_TRUE(obj.GetFieldByName("name", &field_val));
- StringValue sv = field_val.GetString();
+ StringValue sv;
+ ASSERT_TRUE(field_val.GetString(&sv));
EXPECT_EQ(string(sv.Ptr(), sv.Len()), "Alice");
EXPECT_FALSE(obj.GetFieldByName("unknown", &field_val));
@@ -245,13 +256,18 @@ TEST(VariantUtilTest, SimpleArray) {
VariantValue arr(arr_bytes.data(), arr_bytes.size(), &metadata);
EXPECT_EQ(arr.GetBasicType(), VariantBasicType::ARRAY);
- EXPECT_EQ(3, arr.GetArraySize());
+ uint32_t arr_size;
+ EXPECT_TRUE(arr.GetArraySize(&arr_size));
+ EXPECT_EQ(3, arr_size);
VariantValue elem;
+ int32_t elem_int;
EXPECT_TRUE(arr.GetArrayElement(0, &elem));
- EXPECT_EQ(1, elem.GetInt32());
+ EXPECT_TRUE(elem.GetInt32(&elem_int));
+ EXPECT_EQ(1, elem_int);
EXPECT_TRUE(arr.GetArrayElement(2, &elem));
- EXPECT_EQ(3, elem.GetInt32());
+ EXPECT_TRUE(elem.GetInt32(&elem_int));
+ EXPECT_EQ(3, elem_int);
EXPECT_FALSE(arr.GetArrayElement(3, &elem));
}
@@ -270,11 +286,14 @@ TEST(VariantUtilTest, PathNavigation) {
VariantValue root(outer_obj.data(), outer_obj.size(), &metadata);
VariantValue result;
+ int32_t nav_int;
EXPECT_TRUE(root.NavigatePath("$.data.items[1]", &result));
- EXPECT_EQ(20, result.GetInt32());
+ EXPECT_TRUE(result.GetInt32(&nav_int));
+ EXPECT_EQ(20, nav_int);
EXPECT_TRUE(root.NavigatePath("$.data.items[2]", &result));
- EXPECT_EQ(30, result.GetInt32());
+ EXPECT_TRUE(result.GetInt32(&nav_int));
+ EXPECT_EQ(30, nav_int);
EXPECT_FALSE(root.NavigatePath("$.data.unknown", &result));
EXPECT_FALSE(root.NavigatePath("$.data.items[100]", &result));
@@ -603,14 +622,19 @@ TEST(VariantUtilTest, ObjectDifferentFieldIdAndOffsetSize) {
VariantValue obj(obj_buf.data(), obj_buf.size(), &metadata);
EXPECT_EQ(obj.GetBasicType(), VariantBasicType::OBJECT);
- EXPECT_EQ(2, obj.GetObjectSize());
+ uint32_t obj_size;
+ EXPECT_TRUE(obj.GetObjectSize(&obj_size));
+ EXPECT_EQ(2, obj_size);
VariantValue field_val;
+ int32_t field_int;
EXPECT_TRUE(obj.GetFieldByName("f0", &field_val));
- EXPECT_EQ(42, field_val.GetInt32());
+ EXPECT_TRUE(field_val.GetInt32(&field_int));
+ EXPECT_EQ(42, field_int);
EXPECT_TRUE(obj.GetFieldByName("f299", &field_val));
- EXPECT_EQ(7, field_val.GetInt32());
+ EXPECT_TRUE(field_val.GetInt32(&field_int));
+ EXPECT_EQ(7, field_int);
EXPECT_FALSE(obj.GetFieldByName("f1", &field_val));
}
@@ -638,7 +662,8 @@ TEST(VariantUtilTest, LongStringPrimitive) {
VariantValue val(val_buf.data(), val_buf.size(), &metadata);
EXPECT_EQ(val.GetBasicType(), VariantBasicType::PRIMITIVE);
EXPECT_EQ(val.GetPhysicalType(), VariantPhysicalType::STRING);
- StringValue sv = val.GetString();
+ StringValue sv;
+ ASSERT_TRUE(val.GetString(&sv));
EXPECT_EQ(string(sv.Ptr(), sv.Len()), long_str);
}
@@ -823,4 +848,158 @@ TEST(VariantUtilTest, DuckDbVariants) {
}
}
+// --- Corrupt / malformed input handling ---
+
+// Expects VariantToJson to reject the (metadata, value) blobs with an error rather than
+// reading out of bounds.
+static void ExpectVariantError(const vector<uint8_t>& metadata,
+ const vector<uint8_t>& value) {
+ string json;
+ Status s = VariantToJson(metadata.data(), metadata.size(),
+ value.data(), value.size(), &json);
+ EXPECT_FALSE(s.ok()) << "expected an error but succeeded with json: " << json;
+}
+
+// Expects VariantMetadata::Init to reject 'metadata' as malformed.
+static void ExpectMetadataError(const vector<uint8_t>& metadata) {
+ VariantMetadata m;
+ EXPECT_FALSE(m.Init(metadata.data(), metadata.size()).ok());
+}
+
+TEST(VariantUtilTest, MetadataCorruptionRejected) {
+ // Empty blob.
+ ExpectMetadataError({});
+ // Unsupported version (version=2).
+ ExpectMetadataError({0x02, 0x00, 0x00});
+ // offset_size=4 (header bits 6-7 = 3) but blob too short to hold the dictionary size.
+ ExpectMetadataError({0xC1, 0x00});
+ // dict_size=3 but no offset array / string data present.
+ ExpectMetadataError({0x01, 0x03});
+ // Integer-overflow attempt: offset_size=4, dict_size=0xFFFFFFFF, short blob. In 32-bit
+ // math (dict_size+1)*offset_size wraps to 0, so the check would wrongly pass; the
+ // 64-bit math must reject it.
+ ExpectMetadataError({0xC1, 0xFF, 0xFF, 0xFF, 0xFF});
+ // Non-monotonic dictionary offsets [0, 5, 3].
+ ExpectMetadataError({0x01, 0x02, 0x00, 0x05, 0x03, 'a', 'b', 'c', 'd', 'e'});
+ // Last offset (10) exceeds the 2-byte string-data region.
+ ExpectMetadataError({0x01, 0x01, 0x00, 0x0A, 'a', 'b'});
+
+ // Valid baseline still parses.
+ VariantMetadata m;
+ const vector<uint8_t> valid = {0x01, 0x01, 0x00, 0x03, 'a', 'b', 'c'};
+ EXPECT_OK(m.Init(valid.data(), valid.size()));
+ EXPECT_EQ(m.GetFieldName(0), "abc");
+}
+
+TEST(VariantUtilTest, ValueCorruptionRejected) {
+ const vector<uint8_t> empty_meta = {0x01, 0x00, 0x00};
+ // Metadata dictionary [a, b, c, d, e] (sorted).
+ const vector<uint8_t> meta5 = {0x11, 0x05, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x61, 0x62, 0x63, 0x64, 0x65};
+
+ // Empty value buffer.
+ ExpectVariantError(empty_meta, {});
+ // Short string claims length 5 but has only 2 payload bytes.
+ ExpectVariantError(empty_meta, {0x15, 'a', 'b'});
+ // Long string (STRING=0x40) with a 4-byte length of 0xFFFFFFFF and no payload.
+ ExpectVariantError(empty_meta, {0x40, 0xFF, 0xFF, 0xFF, 0xFF});
+ // Long string with a truncated (2-byte) length prefix.
+ ExpectVariantError(empty_meta, {0x40, 0x01, 0x00});
+ // INT8 (0x0c) with no payload byte (needs 1).
+ ExpectVariantError(empty_meta, {0x0c});
+ // INT16 (0x10) with only 1 payload byte (needs 2).
+ ExpectVariantError(empty_meta, {0x10, 0x01});
+ // INT32 (0x14) with only 2 payload bytes (needs 4).
+ ExpectVariantError(empty_meta, {0x14, 0x01, 0x02});
+ // INT64 (0x18) with only 4 payload bytes (needs 8).
+ ExpectVariantError(empty_meta, {0x18, 0x01, 0x02, 0x03, 0x04});
+ // FLOAT (0x38) with only 2 payload bytes (needs 4).
+ ExpectVariantError(empty_meta, {0x38, 0x01, 0x02});
+ // DOUBLE (0x1c) with only 4 payload bytes (needs 8).
+ ExpectVariantError(empty_meta, {0x1c, 0x01, 0x02, 0x03, 0x04});
+ // Binary (0x3c) claiming a huge payload length.
+ ExpectVariantError(empty_meta, {0x3c, 0xFF, 0xFF, 0xFF, 0x7F});
+ // Object claiming num_fields=100 in a tiny buffer.
+ ExpectVariantError(meta5, {0x02, 0x64, 0x00});
+ // Object with a field offset (99) past the end of the data region.
+ ExpectVariantError(meta5, {0x02, 0x01, 0x00, 0x00, 0x63});
+ // Object with non-monotonic field offsets [5, 0].
+ ExpectVariantError(meta5,
+ {0x02, 0x01, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
+ // Object with a field id (200) that is out of range of the 5-entry dictionary.
+ ExpectVariantError(meta5, {0x02, 0x01, 0xC8, 0x00, 0x01, 0x00});
+ // Array claiming num_elements=200 in a tiny buffer.
+ ExpectVariantError(empty_meta, {0x03, 0xC8, 0x00});
+ // Array with an element offset (50) past the end of the data region.
+ ExpectVariantError(empty_meta, {0x03, 0x01, 0x00, 0x32});
+ // DECIMAL4 (0x20) with scale=255, past the max precision of 9. An unchecked scale would
+ // underflow the output buffer in DecimalValue::ToString (OOB write).
+ ExpectVariantError(empty_meta, {0x20, 0xFF, 0x00, 0x00, 0x00, 0x00});
+ // DECIMAL4 with scale=10, one past the max precision of 9.
+ ExpectVariantError(empty_meta, {0x20, 0x0A, 0x00, 0x00, 0x00, 0x00});
+ // DECIMAL8 (0x24) with scale=200, past the max precision of 18.
+ ExpectVariantError(empty_meta,
+ {0x24, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
+ // DECIMAL16 (0x28) with scale=255, past the max precision of 38.
+ ExpectVariantError(empty_meta,
+ {0x28, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
+ // Truncated fixed-width payloads: the scale byte is present but the unscaled value is
+ // cut short (DECIMAL4/8/16 need 4/8/16 bytes past the scale byte).
+ ExpectVariantError(empty_meta, {0x20, 0x02, 0x01});
+ ExpectVariantError(empty_meta, {0x24, 0x02, 0x01, 0x02, 0x03, 0x04});
+ ExpectVariantError(empty_meta,
+ {0x28, 0x02, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08});
+ // DATE (0x2c) whose day count (0x7FFFFFFF) is far outside the [0001-01-01, 9999-12-31]
+ // range: structurally well-formed but unrepresentable, so it is rejected like any other
+ // malformed value rather than silently emitting a placeholder string.
+ ExpectVariantError(empty_meta, {0x2c, 0xFF, 0xFF, 0xFF, 0x7F});
+ // DATE (0x2c) with only 1 payload byte (needs 4).
+ ExpectVariantError(empty_meta, {0x2c, 0x01});
+ // TIMESTAMPNTZ (0x34) with only 1 payload byte (needs 8).
+ ExpectVariantError(empty_meta, {0x34, 0x01});
+ // TIMESTAMPNTZ_NANOS (0x4c) with only 1 payload byte (needs 8).
+ ExpectVariantError(empty_meta, {0x4c, 0x01});
+}
+
+// A decimal whose scale is exactly the max precision (the boundary of the valid range)
+// must still serialize successfully.
+TEST(VariantUtilTest, DecimalScaleBoundaryAccepted) {
+ const vector<uint8_t> empty_meta = {0x01, 0x00, 0x00};
+ string json;
+ // DECIMAL4 with scale=9 (== MAX_DECIMAL4_PRECISION), unscaled=0.
+ vector<uint8_t> dec4 = {0x20, 0x09, 0x00, 0x00, 0x00, 0x00};
+ EXPECT_OK(VariantToJson(empty_meta.data(), empty_meta.size(),
+ dec4.data(), dec4.size(), &json));
+}
+
+// Builds a value that wraps 'inner' in 'depth' nested single-element arrays. Uses 2-byte
+// offsets so the growing child sizes stay representable as the nesting deepens.
+static vector<uint8_t> WrapInArrays(vector<uint8_t> inner, int depth) {
+ for (int i = 0; i < depth; ++i) {
+ uint32_t child_size = inner.size();
+ // header 0x07 = ARRAY with 2-byte offsets; num_elems=1; offsets [0, child_size].
+ vector<uint8_t> a = {0x07, 0x01, 0x00, 0x00,
+ static_cast<uint8_t>(child_size & 0xFF),
+ static_cast<uint8_t>((child_size >> 8) & 0xFF)};
+ a.insert(a.end(), inner.begin(), inner.end());
+ inner.swap(a);
+ }
+ return inner;
+}
+
+TEST(VariantUtilTest, NestingDepthLimit) {
+ const vector<uint8_t> empty_meta = {0x01, 0x00, 0x00};
+
+ // A structurally valid but excessively deep value must be rejected (not overflow the
+ // stack) when serialized.
+ ExpectVariantError(empty_meta, WrapInArrays({0x00}, 105));
+
+ // A moderately nested value within the cap serializes fine.
+ vector<uint8_t> ok = WrapInArrays({0x00}, 50);
+ string json;
+ EXPECT_OK(VariantToJson(empty_meta.data(), empty_meta.size(),
+ ok.data(), ok.size(), &json));
+}
+
} // namespace impala
diff --git a/fe/src/main/java/org/apache/impala/catalog/Type.java b/fe/src/main/java/org/apache/impala/catalog/Type.java
index 9061fe735..43dbfae11 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Type.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Type.java
@@ -47,6 +47,7 @@ public abstract class Type {
// generating and scanning deeply nested Parquet and Avro files. In those experiments,
// we exceeded the stack space in the scanner (which uses recursion for dealing with
// nested types) at a nesting depth between 200 and 300 (200 worked, 300 crashed).
+ // Must be kept in sync with the backend's ColumnType::MAX_NESTING_DEPTH.
public static int MAX_NESTING_DEPTH = 100;
// Static constant types for scalar types that don't require additional information.