You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2026/07/21 12:47:49 UTC
(asterixdb) branch master updated: [NO ISSUE][EXT]: fix parsing nulls and handling time micros for avro
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 62a6aa125f [NO ISSUE][EXT]: fix parsing nulls and handling time micros for avro
62a6aa125f is described below
commit 62a6aa125fe51b6ff13c85f778da4c31102f0f6b
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Wed Jun 17 21:25:47 2026 +0300
[NO ISSUE][EXT]: fix parsing nulls and handling time micros for avro
Details:
- Properly report Avro NULL value as null rather than missing and
omitting the whole field.
- Stop applying the timezone offset to time-of-day and local
timestamp values, which have no associated timezone; only
UTC timestamp-millis/micros values get the offset.
Ext-ref: MB-72320, MB-72680
Change-Id: I872be48898e01e7eefe4f1e2c85e72f1ca0c9c2e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21389
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../avro/AvroLogicalTypesExampleGenerator.java | 12 +++-
.../avro-logical-types.01.ddl.sqlpp | 15 +++++
...ddl.sqlpp => avro-logical-types.04.query.sqlpp} | 36 +----------
.../common/avro/array-access/array-access.02.adm | 2 +-
.../common/avro/array-access/array-access.03.adm | 1 +
.../avro-logical-types/avro-logical-types.02.adm | 2 +-
.../avro-logical-types/avro-logical-types.03.adm | 2 +-
...ical-types.03.adm => avro-logical-types.04.adm} | 2 +-
.../avro/select-all-fields/select-all-fields.2.adm | 2 +-
.../common/avro/type-mismatch/type-mismatch.02.adm | 2 +-
.../asterix/external/parser/AvroDataParser.java | 71 +++++++++++++++-------
11 files changed, 82 insertions(+), 65 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
index fd546279f2..0015c5b6d4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/avro/AvroLogicalTypesExampleGenerator.java
@@ -29,6 +29,7 @@ import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.Test;
public class AvroLogicalTypesExampleGenerator {
@@ -42,12 +43,15 @@ public class AvroLogicalTypesExampleGenerator {
+ " { \"name\": \"timestampMillisField\", \"type\": { \"type\": \"long\", \"logicalType\": \"timestamp-millis\" } },\n"
+ " { \"name\": \"timestampMicrosField\", \"type\": { \"type\": \"long\", \"logicalType\": \"timestamp-micros\" } },\n"
+ " { \"name\": \"localTimestampMillisField\", \"type\": { \"type\": \"long\", \"logicalType\": \"local-timestamp-millis\" } },\n"
- + " { \"name\": \"localTimestampMicrosField\", \"type\": { \"type\": \"long\", \"logicalType\": \"local-timestamp-micros\" } }\n"
- + " ]\n" + "}";
+ + " { \"name\": \"localTimestampMicrosField\", \"type\": { \"type\": \"long\", \"logicalType\": \"local-timestamp-micros\" } },\n"
+ + " { \"name\": \"nullableField\", \"type\": [\"null\", \"string\"], \"default\": null }\n" + " ]\n"
+ + "}";
private static final String AVRO_GEN_BASEDIR = "target/generated_avro_files";
private static final String FILE_NAME = "avro_logical_type.avro";
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Fix timeMicrosField test value overflowing a day (was 12345678901234, > 24h in micros), which silently overflowed int during Avro TimeMicros parsing")
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_4_6, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Add a null-valued nullable field so tests can cover Avro NULL (not MISSING) parsing")
public static void writeLogicalTypesExample() throws IOException {
Schema schema = new Schema.Parser().parse(SCHEMA_STRING);
File destPath = new File(AVRO_GEN_BASEDIR);
@@ -66,11 +70,13 @@ public class AvroLogicalTypesExampleGenerator {
record.put("uuidField", "123e4567-e89b-12d3-a456-426614174000");
record.put("dateField", 20061);
record.put("timeMillisField", 12345678);
- record.put("timeMicrosField", 12345678901234L);
+ // time-micros is microseconds after midnight (< 86,400,000,000); 45296789012 = 12:34:56.789012
+ record.put("timeMicrosField", 45296789012L);
record.put("timestampMillisField", 1733344079083L);
record.put("timestampMicrosField", 1733344079083000L);
record.put("localTimestampMillisField", 1733344079083L);
record.put("localTimestampMicrosField", 1733344079083000L);
+ record.put("nullableField", null);
dataFileWriter.append(record);
} catch (IOException e) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
index 73d8056628..ca8294d8f7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
@@ -53,4 +53,19 @@ CREATE EXTERNAL DATASET AvroDataset2(AvroType) USING %adapter%
("time-to-long"="false"),
("timestamp-to-long"="false"),
("format" = "avro")
+);
+
+CREATE EXTERNAL DATASET AvroDataset3(AvroType) USING %adapter%
+(
+ %template%,
+ ("container"="playground"),
+ ("definition"="avro-data/reviews"),
+ ("include"="*avro_logical_type.avro"),
+ ("decimal-to-double"="true"),
+ ("uuid-to-string"="false"),
+ ("date-to-int"="false"),
+ ("time-to-long"="false"),
+ ("timestamp-to-long"="false"),
+ ("timezone"="PST"),
+ ("format" = "avro")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
similarity index 50%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
index 73d8056628..2ac7e29da5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.query.sqlpp
@@ -17,40 +17,8 @@
* under the License.
*/
-
-DROP DATAVERSE test IF EXISTS;
-CREATE DATAVERSE test;
-
USE test;
-CREATE TYPE AvroType as {
-};
-
-CREATE EXTERNAL DATASET AvroDataset(AvroType) USING %adapter%
-(
- %template%,
- ("container"="playground"),
- ("definition"="avro-data/reviews"),
- ("include"="*avro_logical_type.avro"),
- ("decimal-to-double"="true"),
- ("uuid-to-string"="true"),
- ("date-to-int"="true"),
- ("time-to-long"="true"),
- ("timestamp-to-long"="true"),
- ("format" = "avro")
-);
-
-CREATE EXTERNAL DATASET AvroDataset2(AvroType) USING %adapter%
-(
- %template%,
- ("container"="playground"),
- ("definition"="avro-data/reviews"),
- ("include"="*avro_logical_type.avro"),
- ("decimal-to-double"="true"),
- ("uuid-to-string"="false"),
- ("date-to-int"="false"),
- ("time-to-long"="false"),
- ("timestamp-to-long"="false"),
- ("format" = "avro")
-);
\ No newline at end of file
+SELECT VALUE a
+FROM AvroDataset3 a;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
index 18f3275beb..ff1820c120 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.02.adm
@@ -1,2 +1,2 @@
{ "display_url": "string" }
-{ }
+{ "display_url": null }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
index 695240b26c..113982075f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/array-access/array-access.03.adm
@@ -1 +1,2 @@
{ "display_url": [ "string" ] }
+{ "display_url": null }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
index 7690b33163..6d2a06cfb2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.02.adm
@@ -1 +1 @@
-{ "decimalField": 2.58, "uuidField": "123e4567-e89b-12d3-a456-426614174000", "dateField": 20061, "timeMillisField": 12345678, "timeMicrosField": -539222987, "timestampMillisField": 1733344079083, "timestampMicrosField": 1733344079083, "localTimestampMillisField": 1733344079083, "localTimestampMicrosField": 1733344079083 }
\ No newline at end of file
+{ "decimalField": 2.58, "uuidField": "123e4567-e89b-12d3-a456-426614174000", "dateField": 20061, "timeMillisField": 12345678, "timeMicrosField": 45296789, "timestampMillisField": 1733344079083, "timestampMicrosField": 1733344079083, "localTimestampMillisField": 1733344079083, "localTimestampMicrosField": 1733344079083, "nullableField": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
index 51eedaef4f..b01df9c259 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
@@ -1 +1 @@
-{ "decimalField": 2.58, "uuidField": uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"), "timeMillisField": time("03:25:45.678"), "timeMicrosField": time("18:12:57.013"), "timestampMillisField": datetime("2024-12-04T20:27:59.083"), "timestampMicrosField": datetime("2024-12-04T20:27:59.083"), "localTimestampMillisField": datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField": datetime("2024-12-04T20:27:59.083") }
\ No newline at end of file
+{ "decimalField": 2.58, "uuidField": uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"), "timeMillisField": time("03:25:45.678"), "timeMicrosField": time("12:34:56.789"), "timestampMillisField": datetime("2024-12-04T20:27:59.083"), "timestampMicrosField": datetime("2024-12-04T20:27:59.083"), "localTimestampMillisField": datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField": datetime("2024-12-04T20:27:59.083"), "nullableField": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
similarity index 53%
copy from asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
copy to asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
index 51eedaef4f..dbfb24c4d0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.03.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/avro-logical-types/avro-logical-types.04.adm
@@ -1 +1 @@
-{ "decimalField": 2.58, "uuidField": uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"), "timeMillisField": time("03:25:45.678"), "timeMicrosField": time("18:12:57.013"), "timestampMillisField": datetime("2024-12-04T20:27:59.083"), "timestampMicrosField": datetime("2024-12-04T20:27:59.083"), "localTimestampMillisField": datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField": datetime("2024-12-04T20:27:59.083") }
\ No newline at end of file
+{ "decimalField": 2.58, "uuidField": uuid("123e4567-e89b-12d3-a456-426614174000"), "dateField": date("2024-12-04"), "timeMillisField": time("03:25:45.678"), "timeMicrosField": time("12:34:56.789"), "timestampMillisField": datetime("2024-12-04T12:27:59.083"), "timestampMicrosField": datetime("2024-12-04T12:27:59.083"), "localTimestampMillisField": datetime("2024-12-04T20:27:59.083"), "localTimestampMicrosField": datetime("2024-12-04T20:27:59.083"), "nullableField": null }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
index 53f2518eb9..878118985e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/select-all-fields/select-all-fields.2.adm
@@ -1,2 +1,2 @@
{ "coordinates": { "coordinates": [ 1.1 ], "type": "string" }, "created_at": "string", "entities": { "urls": [ { "display_url": "string", "expanded_url": "string", "indices": [ 1 ], "url": "string" } ], "user_mentions": [ { "id": 1, "id_str": "string", "indices": [ 1 ], "name": "string", "screen_name": "string" } ] }, "favorite_count": 1, "favorited": true, "filter_level": "string", "geo": { "coordinates": [ 1.1 ], "type": "string" }, "id": "0000000", "id_str": "string", "in_reply_to_scr [...]
-{ "coordinates": { "coordinates": [ 1.1 ], "type": "string" }, "created_at": "string", "favorite_count": 1, "favorited": true, "filter_level": "string", "geo": { "coordinates": [ 1.1 ], "type": "string" }, "id": "11111111111111111111", "id_str": "string", "in_reply_to_screen_name": "string", "in_reply_to_status_id": 1, "in_reply_to_status_id_str": "string", "in_reply_to_user_id": 1, "in_reply_to_user_id_str": "string", "is_quote_status": true, "lang": "string", "place": { "bounding_box": [...]
+{ "coordinates": { "coordinates": [ 1.1 ], "type": "string" }, "created_at": "string", "entities": null, "favorite_count": 1, "favorited": true, "filter_level": "string", "geo": { "coordinates": [ 1.1 ], "type": "string" }, "id": "11111111111111111111", "id_str": "string", "in_reply_to_screen_name": "string", "in_reply_to_status_id": 1, "in_reply_to_status_id_str": "string", "in_reply_to_user_id": 1, "in_reply_to_user_id_str": "string", "is_quote_status": true, "lang": "string", "place": [...]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
index 0be5d980da..d25232800f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/avro/type-mismatch/type-mismatch.02.adm
@@ -1,2 +1,2 @@
true
-true
\ No newline at end of file
+false
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
index faf6d7291e..81f47e991e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AvroDataParser.java
@@ -55,6 +55,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.util.annotations.AiProvenance;
public class AvroDataParser extends AbstractDataParser implements IRecordDataParser<GenericRecord> {
private final AvroConverterContext parserContext;
@@ -77,6 +78,7 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Add the field even when its value is NULL (previously only non-null/non-missing values were added, silently dropping null fields)")
private void parseObject(GenericRecord record, DataOutput out) throws IOException {
IMutableValueStorage valueBuffer = parserContext.enterObject();
IARecordBuilder objectBuilder = parserContext.getObjectBuilder(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
@@ -88,17 +90,16 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
Object fieldValue = record.get(fieldName);
ATypeTag typeTag = getTypeTag(fieldSchema, fieldValue);
- IValueReference value = null;
+ IValueReference value;
if (valueEmbedder.shouldEmbed(fieldName, typeTag)) {
value = valueEmbedder.getEmbeddedValue();
- } else if (fieldValue != null) {
+ } else {
valueBuffer.reset();
parseValue(fieldSchema, fieldValue, valueBuffer.getDataOutput());
value = valueBuffer;
}
- if (value != null) {
- // Ignore missing values
+ if (value != null || typeTag == ATypeTag.NULL) {
objectBuilder.addField(parserContext.getSerializedFieldName(fieldName), value);
}
}
@@ -148,26 +149,30 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
parserContext.exitCollection(item, listBuilder);
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Throw a well-formed unsupported-type error instead of a 1-arg message that crashes formatting")
private void parseUnion(Schema unionSchema, Object value, DataOutput out) throws IOException {
Schema actualSchema = getActualSchema(unionSchema, value);
if (actualSchema != null) {
parseValue(actualSchema, value, out);
} else {
- throw new RuntimeDataException(ErrorCode.TYPE_UNSUPPORTED, unionSchema.getType());
+ throw createUnsupportedException(unionSchema);
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Return the null branch of a nullable union for null values instead of returning null (no match)")
private Schema getActualSchema(Schema unionSchema, Object value) {
List<Schema> possibleTypes = unionSchema.getTypes();
+ Schema nullSchema = null;
for (Schema possibleType : possibleTypes) {
Schema.Type schemaType = possibleType.getType();
- if (schemaType != NULL) {
- if (matchesType(value, schemaType)) {
- return possibleType;
- }
+ if (schemaType == NULL) {
+ nullSchema = possibleType;
+ } else if (matchesType(value, schemaType)) {
+ return possibleType;
}
}
- return null;
+ // value is null — return the null branch of the union if present
+ return value == null ? nullSchema : null;
}
private boolean matchesType(Object value, Schema.Type schemaType) {
@@ -197,6 +202,7 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Map null values to NULL instead of MISSING; make union fall-through explicit")
private ATypeTag getTypeTag(Schema schema, Object value) throws HyracksDataException {
Schema.Type schemaType = schema.getType();
LogicalType logicalType = schema.getLogicalType();
@@ -235,8 +241,9 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
}
if (value == null) {
- // The 'value' is missing
- return ATypeTag.MISSING;
+ // A null value (e.g. the null branch of a nullable union) maps to NULL, not MISSING.
+ // Avro records always carry every declared field, so there is no true "missing" here.
+ return ATypeTag.NULL;
}
switch (schemaType) {
@@ -264,12 +271,15 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
if (actualSchema != null) {
return getTypeTag(actualSchema, value);
}
+ throw createUnsupportedException(schema);
default:
throw createUnsupportedException(schema);
-
}
}
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_SONNET_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Only apply the timezone offset to UTC-instant timestamps (timestamp-millis/micros); "
+ + "stop double-applying it to time-micros, and stop applying it at all to time-millis/micros "
+ + "and local-timestamp-millis/micros, none of which carry an associated timezone")
private void parseLogicalValue(LogicalType logicalType, Object value, DataOutput out) throws IOException {
if (logicalType instanceof LogicalTypes.Uuid) {
if (parserContext.isUuidAsString()) {
@@ -288,25 +298,24 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
parserContext.serializeDate(value, out);
}
} else if (logicalType instanceof LogicalTypes.TimeMicros) {
+ // time-micros is a time of day with no reference to any timezone; it must not be shifted
+ // by the configured offset (unlike timestamp-millis/micros, which are UTC instants).
int timeInMillis = (int) TimeUnit.MICROSECONDS.toMillis(((Number) value).longValue());
- int offset = parserContext.getTimeZoneOffset();
- timeInMillis = timeInMillis + offset;
if (parserContext.isTimeAsLong()) {
serializeLong(timeInMillis, out);
} else {
- parserContext.serializeTime(timeInMillis + offset, out);
+ parserContext.serializeTime(timeInMillis, out);
}
} else if (logicalType instanceof LogicalTypes.TimeMillis) {
+ // time-millis is a time of day with no reference to any timezone; it must not be shifted
+ // by the configured offset (unlike timestamp-millis/micros, which are UTC instants).
int timeInMillis = ((Number) value).intValue();
- int offset = parserContext.getTimeZoneOffset();
- timeInMillis = timeInMillis + offset;
if (parserContext.isTimeAsLong()) {
serializeLong(timeInMillis, out);
} else {
parserContext.serializeTime(timeInMillis, out);
}
- } else if (logicalType instanceof LogicalTypes.TimestampMicros
- || logicalType instanceof LogicalTypes.LocalTimestampMicros) {
+ } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
long timeStampInMicros = ((Number) value).longValue();
int offset = parserContext.getTimeZoneOffset();
long timeStampInMillis = TimeUnit.MICROSECONDS.toMillis(timeStampInMicros);
@@ -316,8 +325,7 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
} else {
parserContext.serializeDateTime(timeStampInMillis, out);
}
- } else if (logicalType instanceof LogicalTypes.TimestampMillis
- || logicalType instanceof LogicalTypes.LocalTimestampMillis) {
+ } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
long timeStampInMillis = ((Number) value).longValue();
int offset = parserContext.getTimeZoneOffset();
timeStampInMillis = timeStampInMillis + offset;
@@ -326,6 +334,25 @@ public class AvroDataParser extends AbstractDataParser implements IRecordDataPar
} else {
parserContext.serializeDateTime(timeStampInMillis, out);
}
+ } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros) {
+ // local-timestamp-micros is already wall-clock time with no associated timezone;
+ // unlike timestamp-micros (a UTC instant), it must not be shifted by the configured offset.
+ long timeStampInMicros = ((Number) value).longValue();
+ long timeStampInMillis = TimeUnit.MICROSECONDS.toMillis(timeStampInMicros);
+ if (parserContext.isTimestampAsLong()) {
+ serializeLong(timeStampInMillis, out);
+ } else {
+ parserContext.serializeDateTime(timeStampInMillis, out);
+ }
+ } else if (logicalType instanceof LogicalTypes.LocalTimestampMillis) {
+ // local-timestamp-millis is already wall-clock time with no associated timezone;
+ // unlike timestamp-millis (a UTC instant), it must not be shifted by the configured offset.
+ long timeStampInMillis = ((Number) value).longValue();
+ if (parserContext.isTimestampAsLong()) {
+ serializeLong(timeStampInMillis, out);
+ } else {
+ parserContext.serializeDateTime(timeStampInMillis, out);
+ }
} else {
throw createUnsupportedException(logicalType.getName());
}