You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2023/08/03 02:52:09 UTC
[hudi] branch master updated: [HUDI-6569] Fix write failure for Avro Enum type (#9237)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4017b96fb1b [HUDI-6569] Fix write failure for Avro Enum type (#9237)
4017b96fb1b is described below
commit 4017b96fb1bf47283d0d16deea28fb5dc806d8eb
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Wed Aug 2 19:52:02 2023 -0700
[HUDI-6569] Fix write failure for Avro Enum type (#9237)
- Fix a regression for Avro ENUM type.
- Adds logic to handle ENUM type in `HoodieAvroUtils.rewriteRecordWithNewSchemaInternal` and `AvroDeserializer`.
---
.../commit/TestJavaCopyOnWriteActionExecutor.java | 3 +-
.../src/test/resources/testDataGeneratorSchema.txt | 132 ------------
.../commit/TestCopyOnWriteActionExecutor.java | 3 +-
.../GenericRecordValidationTestUtils.java | 5 +
.../src/test/resources/testDataGeneratorSchema.txt | 132 ------------
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 18 +-
.../common/testutils/HoodieTestDataGenerator.java | 9 +-
.../apache/hudi/common/util/TestAvroOrcUtils.java | 5 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 1 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 1 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 1 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 15 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 13 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 1 +
.../streamer-config/source-flattened.avsc | 101 +++++----
.../src/test/resources/streamer-config/source.avsc | 228 +++++++++++---------
.../resources/streamer-config/source_evolved.avsc | 4 +
.../source_evolved_post_processed.avsc | 4 +
.../streamer-config/sql-transformer.properties | 2 +-
.../streamer-config/target-flattened.avsc | 108 ++++++----
.../src/test/resources/streamer-config/target.avsc | 235 ++++++++++++---------
21 files changed, 451 insertions(+), 570 deletions(-)
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
index a272585b360..f57b21d89be 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestJavaCopyOnWriteActionExecutor.java
@@ -408,11 +408,10 @@ public class TestJavaCopyOnWriteActionExecutor extends HoodieJavaClientTestHarne
@Test
public void testInsertUpsertWithHoodieAvroPayload() throws Exception {
- Schema schema = getSchemaFromResource(TestJavaCopyOnWriteActionExecutor.class, "/testDataGeneratorSchema.txt");
HoodieWriteConfig config = HoodieWriteConfig.newBuilder()
.withEngineType(EngineType.JAVA)
.withPath(basePath)
- .withSchema(schema.toString())
+ .withSchema(TRIP_EXAMPLE_SCHEMA)
.withStorageConfig(HoodieStorageConfig.newBuilder()
.parquetMaxFileSize(1000 * 1024).hfileMaxFileSize(1000 * 1024).build())
.build();
diff --git a/hudi-client/hudi-java-client/src/test/resources/testDataGeneratorSchema.txt b/hudi-client/hudi-java-client/src/test/resources/testDataGeneratorSchema.txt
deleted file mode 100644
index c80365b76ea..00000000000
--- a/hudi-client/hudi-java-client/src/test/resources/testDataGeneratorSchema.txt
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-{
- "type" : "record",
- "name" : "triprec",
- "fields" : [
- {
- "name" : "timestamp",
- "type" : "long"
- }, {
- "name" : "_row_key",
- "type" : "string"
- }, {
- "name" : "partition_path",
- "type" : ["null", "string"],
- "default": null
- }, {
- "name" : "rider",
- "type" : "string"
- }, {
- "name" : "driver",
- "type" : "string"
- }, {
- "name" : "begin_lat",
- "type" : "double"
- }, {
- "name" : "begin_lon",
- "type" : "double"
- }, {
- "name" : "end_lat",
- "type" : "double"
- }, {
- "name" : "end_lon",
- "type" : "double"
- }, {
- "name" : "distance_in_meters",
- "type" : "int"
- }, {
- "name" : "seconds_since_epoch",
- "type" : "long"
- }, {
- "name" : "weight",
- "type" : "float"
- },{
- "name" : "nation",
- "type" : "bytes"
- },{
- "name" : "current_date",
- "type" : {
- "type" : "int",
- "logicalType" : "date"
- }
- },{
- "name" : "current_ts",
- "type" : {
- "type" : "long"
- }
- },{
- "name" : "height",
- "type" : {
- "type" : "fixed",
- "name" : "abc",
- "size" : 5,
- "logicalType" : "decimal",
- "precision" : 10,
- "scale": 6
- }
- }, {
- "name" :"city_to_state",
- "type" : {
- "type" : "map",
- "values": "string"
- }
- },
- {
- "name" : "fare",
- "type" : {
- "type" : "record",
- "name" : "fare",
- "fields" : [
- {
- "name" : "amount",
- "type" : "double"
- },
- {
- "name" : "currency",
- "type" : "string"
- }
- ]
- }
- },
- {
- "name" : "tip_history",
- "type" : {
- "type" : "array",
- "items" : {
- "type" : "record",
- "name" : "tip_history",
- "fields" : [
- {
- "name" : "amount",
- "type" : "double"
- },
- {
- "name" : "currency",
- "type" : "string"
- }
- ]
- }
- }
- },
- {
- "name" : "_hoodie_is_deleted",
- "type" : "boolean",
- "default" : false
- } ]
-}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
index 548a36dd022..4997ddd5f7c 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
@@ -450,8 +450,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase implemen
@Test
public void testInsertUpsertWithHoodieAvroPayload() throws Exception {
- Schema schema = getSchemaFromResource(TestCopyOnWriteActionExecutor.class, "/testDataGeneratorSchema.txt");
- HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(schema.toString())
+ HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(TRIP_EXAMPLE_SCHEMA)
.withFileSystemViewConfig(FileSystemViewStorageConfig.newBuilder()
.withRemoteServerPort(timelineServicePort).build())
.withStorageConfig(HoodieStorageConfig.newBuilder()
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
index 6b2fa59564a..2196b6f0b63 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
@@ -30,6 +30,8 @@ import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import java.nio.file.Paths;
@@ -45,6 +47,7 @@ import static org.apache.hudi.common.model.HoodieRecord.FILENAME_METADATA_FIELD;
import static org.apache.hudi.common.model.HoodieRecord.OPERATION_METADATA_FIELD;
import static org.apache.hudi.common.model.HoodieRecord.RECORD_KEY_METADATA_FIELD;
import static org.apache.hudi.hadoop.utils.HoodieHiveUtils.HOODIE_CONSUME_COMMIT;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -63,6 +66,8 @@ public class GenericRecordValidationTestUtils {
if (value1 instanceof ArrayWritable) {
assertEquals(HoodieRealtimeRecordReaderUtils.arrayWritableToString((ArrayWritable) value1),
HoodieRealtimeRecordReaderUtils.arrayWritableToString((ArrayWritable) value2));
+ } else if (value1 instanceof Text && value2 instanceof BytesWritable) {
+ assertArrayEquals(((Text) value1).getBytes(), ((BytesWritable) value2).getBytes());
} else {
assertEquals(value1, value2, "Field name " + fieldName + " is not same."
+ " Val1: " + value1 + ", Val2:" + value2);
diff --git a/hudi-client/hudi-spark-client/src/test/resources/testDataGeneratorSchema.txt b/hudi-client/hudi-spark-client/src/test/resources/testDataGeneratorSchema.txt
deleted file mode 100644
index 4cecd7974d5..00000000000
--- a/hudi-client/hudi-spark-client/src/test/resources/testDataGeneratorSchema.txt
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-{
- "type" : "record",
- "name" : "triprec",
- "fields" : [
- {
- "name" : "timestamp",
- "type" : "long"
- }, {
- "name" : "_row_key",
- "type" : "string"
- }, {
- "name" : "partition_path",
- "type" : ["null", "string"],
- "default": null
- }, {
- "name" : "rider",
- "type" : "string"
- }, {
- "name" : "driver",
- "type" : "string"
- }, {
- "name" : "begin_lat",
- "type" : "double"
- }, {
- "name" : "begin_lon",
- "type" : "double"
- }, {
- "name" : "end_lat",
- "type" : "double"
- }, {
- "name" : "end_lon",
- "type" : "double"
- }, {
- "name" : "distance_in_meters",
- "type" : "int"
- }, {
- "name" : "seconds_since_epoch",
- "type" : "long"
- }, {
- "name" : "weight",
- "type" : "float"
- },{
- "name" : "nation",
- "type" : "bytes"
- },{
- "name" : "current_date",
- "type" : {
- "type" : "int",
- "logicalType" : "date"
- }
- },{
- "name" : "current_ts",
- "type" : {
- "type" : "long"
- }
- },{
- "name" : "height",
- "type" : {
- "type" : "fixed",
- "name" : "abc",
- "size" : 5,
- "logicalType" : "decimal",
- "precision" : 10,
- "scale": 6
- }
- }, {
- "name" :"city_to_state",
- "type" : {
- "type" : "map",
- "values": "string"
- }
- },
- {
- "name" : "fare",
- "type" : {
- "type" : "record",
- "name" : "fare",
- "fields" : [
- {
- "name" : "amount",
- "type" : "double"
- },
- {
- "name" : "currency",
- "type" : "string"
- }
- ]
- }
- },
- {
- "name" : "tip_history",
- "type" : {
- "type" : "array",
- "items" : {
- "type" : "record",
- "name" : "tip_history",
- "fields" : [
- {
- "name" : "amount",
- "type" : "double"
- },
- {
- "name" : "currency",
- "type" : "string"
- }
- ]
- }
- }
- },
- {
- "name" : "_hoodie_is_deleted",
- "type" : "boolean",
- "default" : false
- } ]
-}
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
index b7651b1a18d..0909ee5555a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -912,9 +912,17 @@ public class HoodieAvroUtils {
fieldNames.pop();
}
return newRecord;
+ case ENUM:
+ ValidationUtils.checkArgument(
+ oldSchema.getType() == Schema.Type.STRING || oldSchema.getType() == Schema.Type.ENUM,
+ "Only ENUM or STRING type can be converted ENUM type");
+ if (oldSchema.getType() == Schema.Type.STRING) {
+ return new GenericData.EnumSymbol(newSchema, oldRecord);
+ }
+ return oldRecord;
case ARRAY:
ValidationUtils.checkArgument(oldRecord instanceof Collection, "cannot rewrite record with different type");
- Collection array = (Collection)oldRecord;
+ Collection array = (Collection) oldRecord;
List<Object> newArray = new ArrayList();
fieldNames.push("element");
for (Object element : array) {
@@ -1059,10 +1067,10 @@ public class HoodieAvroUtils {
if (newSchema.getLogicalType() instanceof LogicalTypes.Decimal) {
// TODO: support more types
if (oldSchema.getType() == Schema.Type.STRING
- || oldSchema.getType() == Schema.Type.DOUBLE
- || oldSchema.getType() == Schema.Type.INT
- || oldSchema.getType() == Schema.Type.LONG
- || oldSchema.getType() == Schema.Type.FLOAT) {
+ || oldSchema.getType() == Schema.Type.DOUBLE
+ || oldSchema.getType() == Schema.Type.INT
+ || oldSchema.getType() == Schema.Type.LONG
+ || oldSchema.getType() == Schema.Type.FLOAT) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) newSchema.getLogicalType();
// due to Java, there will be precision problems in direct conversion, we should use string instead of use double
BigDecimal bigDecimal = new java.math.BigDecimal(oldValue.toString()).setScale(decimal.getScale(), RoundingMode.HALF_UP);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
index 80fb90f6fe3..cd3755d26c8 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
@@ -104,9 +104,14 @@ public class HoodieTestDataGenerator implements AutoCloseable {
{DEFAULT_FIRST_PARTITION_PATH, DEFAULT_SECOND_PARTITION_PATH, DEFAULT_THIRD_PARTITION_PATH};
public static final int DEFAULT_PARTITION_DEPTH = 3;
+ public static final String TRIP_TYPE_ENUM_TYPE =
+ "{\"type\": \"enum\", \"name\": \"TripType\", \"symbols\": [\"UNKNOWN\", \"UBERX\", \"BLACK\"], \"default\": \"UNKNOWN\"}";
+ public static final Schema TRIP_TYPE_ENUM_SCHEMA = new Schema.Parser().parse(TRIP_TYPE_ENUM_TYPE);
+
public static final String TRIP_SCHEMA_PREFIX = "{\"type\": \"record\"," + "\"name\": \"triprec\"," + "\"fields\": [ "
+ "{\"name\": \"timestamp\",\"type\": \"long\"}," + "{\"name\": \"_row_key\", \"type\": \"string\"},"
+ "{\"name\": \"partition_path\", \"type\": [\"null\", \"string\"], \"default\": null },"
+ + "{\"name\": \"trip_type\", \"type\": " + TRIP_TYPE_ENUM_TYPE + "},"
+ "{\"name\": \"rider\", \"type\": \"string\"}," + "{\"name\": \"driver\", \"type\": \"string\"},"
+ "{\"name\": \"begin_lat\", \"type\": \"double\"}," + "{\"name\": \"begin_lon\", \"type\": \"double\"},"
+ "{\"name\": \"end_lat\", \"type\": \"double\"}," + "{\"name\": \"end_lon\", \"type\": \"double\"},";
@@ -143,7 +148,7 @@ public class HoodieTestDataGenerator implements AutoCloseable {
+ "{\"name\":\"driver\",\"type\":\"string\"},{\"name\":\"fare\",\"type\":\"double\"},{\"name\": \"_hoodie_is_deleted\", \"type\": \"boolean\", \"default\": false}]}";
public static final String NULL_SCHEMA = Schema.create(Schema.Type.NULL).toString();
- public static final String TRIP_HIVE_COLUMN_TYPES = "bigint,string,string,string,string,double,double,double,double,int,bigint,float,binary,int,bigint,decimal(10,6),"
+ public static final String TRIP_HIVE_COLUMN_TYPES = "bigint,string,string,string,string,string,double,double,double,double,int,bigint,float,binary,int,bigint,decimal(10,6),"
+ "map<string,string>,struct<amount:double,currency:string>,array<struct<amount:double,currency:string>>,boolean";
@@ -357,6 +362,8 @@ public class HoodieTestDataGenerator implements AutoCloseable {
rec.put("_row_key", rowKey);
rec.put("timestamp", timestamp);
rec.put("partition_path", partitionPath);
+ rec.put("trip_type", new GenericData.EnumSymbol(
+ TRIP_TYPE_ENUM_SCHEMA, rand.nextInt(2) == 0 ? "UBERX" : "BLACK"));
rec.put("rider", riderName);
rec.put("driver", driverName);
rec.put("begin_lat", rand.nextDouble());
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java
index 9f1f0333a36..6c157349974 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java
@@ -41,9 +41,10 @@ public class TestAvroOrcUtils extends HoodieCommonTestHarness {
// the ORC schema is constructed in the order as AVRO_SCHEMA:
// TRIP_SCHEMA_PREFIX, EXTRA_TYPE_SCHEMA, MAP_TYPE_SCHEMA, FARE_NESTED_SCHEMA, TIP_NESTED_SCHEMA, TRIP_SCHEMA_SUFFIX
// The following types are tested:
- // DATE, DECIMAL, LONG, INT, BYTES, ARRAY, RECORD, MAP, STRING, FLOAT, DOUBLE
+ // DATE, DECIMAL, LONG, INT, BYTES, ARRAY, RECORD, MAP, STRING, FLOAT, DOUBLE, ENUM
TypeDescription orcSchema = TypeDescription.fromString("struct<"
- + "timestamp:bigint,_row_key:string,partition_path:string,rider:string,driver:string,begin_lat:double,"
+ + "timestamp:bigint,_row_key:string,partition_path:string,"
+ + "trip_type:string,rider:string,driver:string,begin_lat:double,"
+ "begin_lon:double,end_lat:double,end_lon:double,"
+ "distance_in_meters:int,seconds_since_epoch:bigint,weight:float,nation:binary,"
+ "current_date:date,current_ts:bigint,height:decimal(10,6),"
diff --git a/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 9725fb63f55..d4d4ece2666 100644
--- a/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -132,6 +132,7 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
val bytes = new Array[Byte](s.getByteLength)
System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
UTF8String.fromBytes(bytes)
+ case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
}
updater.set(ordinal, str)
diff --git a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 3433f3a4085..67c5e169a49 100644
--- a/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -150,6 +150,7 @@ private[sql] class AvroDeserializer(rootAvroType: Schema,
val bytes = new Array[Byte](s.getByteLength)
System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
UTF8String.fromBytes(bytes)
+ case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
}
updater.set(ordinal, str)
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 35b4cb9f3b5..1e3651c87ec 100644
--- a/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -154,6 +154,7 @@ private[sql] class AvroDeserializer(rootAvroType: Schema,
val bytes = new Array[Byte](s.getByteLength)
System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
UTF8String.fromBytes(bytes)
+ case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
}
updater.set(ordinal, str)
diff --git a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 0b609330756..81f5b678ea5 100644
--- a/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -17,28 +17,28 @@
package org.apache.spark.sql.avro
-import java.math.BigDecimal
-import java.nio.ByteBuffer
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
import org.apache.avro.Conversions.DecimalConversion
import org.apache.avro.LogicalTypes.{TimestampMicros, TimestampMillis}
import org.apache.avro.Schema.Type._
import org.apache.avro.generic._
import org.apache.avro.util.Utf8
+import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
import org.apache.spark.sql.avro.AvroDeserializer.{RebaseSpec, createDateRebaseFuncInRead, createTimestampRebaseFuncInRead}
import org.apache.spark.sql.avro.AvroUtils.{toFieldDescription, toFieldStr}
-import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime}
import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime}
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
import org.apache.spark.sql.execution.datasources.DataSourceUtils
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
+import java.math.BigDecimal
+import java.nio.ByteBuffer
import java.util.TimeZone
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
/**
* A deserializer to deserialize data in avro format to data in catalyst format.
@@ -167,6 +167,7 @@ private[sql] class AvroDeserializer(rootAvroType: Schema,
val bytes = new Array[Byte](s.getByteLength)
System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
UTF8String.fromBytes(bytes)
+ case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
}
updater.set(ordinal, str)
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 5e7bab3e51f..f3b22d4d6f3 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -17,27 +17,27 @@
package org.apache.spark.sql.avro
-import java.math.BigDecimal
-import java.nio.ByteBuffer
-import scala.collection.JavaConverters._
-import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
import org.apache.avro.Conversions.DecimalConversion
import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis}
import org.apache.avro.Schema.Type._
import org.apache.avro.generic._
import org.apache.avro.util.Utf8
+import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
import org.apache.spark.sql.avro.AvroDeserializer.{RebaseSpec, createDateRebaseFuncInRead, createTimestampRebaseFuncInRead}
import org.apache.spark.sql.avro.AvroUtils.{AvroMatchedField, toFieldStr}
-import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime}
import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData, RebaseDateTime}
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
import org.apache.spark.sql.execution.datasources.DataSourceUtils
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
+import java.math.BigDecimal
+import java.nio.ByteBuffer
import java.util.TimeZone
+import scala.collection.JavaConverters._
/**
* A deserializer to deserialize data in avro format to data in catalyst format.
@@ -181,6 +181,7 @@ private[sql] class AvroDeserializer(rootAvroType: Schema,
val bytes = new Array[Byte](s.getByteLength)
System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
UTF8String.fromBytes(bytes)
+ case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
}
updater.set(ordinal, str)
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 5e7bab3e51f..1f8a79d26ac 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -181,6 +181,7 @@ private[sql] class AvroDeserializer(rootAvroType: Schema,
val bytes = new Array[Byte](s.getByteLength)
System.arraycopy(s.getBytes, 0, bytes, 0, s.getByteLength)
UTF8String.fromBytes(bytes)
+ case s: GenericData.EnumSymbol => UTF8String.fromString(s.toString)
}
updater.set(ordinal, str)
diff --git a/hudi-utilities/src/test/resources/streamer-config/source-flattened.avsc b/hudi-utilities/src/test/resources/streamer-config/source-flattened.avsc
index d03178ab315..fc44f07163f 100644
--- a/hudi-utilities/src/test/resources/streamer-config/source-flattened.avsc
+++ b/hudi-utilities/src/test/resources/streamer-config/source-flattened.avsc
@@ -19,42 +19,67 @@
"type" : "record",
"name" : "triprec",
"fields" : [
- {
- "name" : "timestamp",
- "type" : "long"
- }, {
- "name" : "_row_key",
- "type" : "string"
- }, {
- "name" : "partition_path",
- "type" : "string"
- }, {
- "name" : "rider",
- "type" : "string"
- }, {
- "name" : "driver",
- "type" : "string"
- }, {
- "name" : "begin_lat",
- "type" : "double"
- }, {
- "name" : "begin_lon",
- "type" : "double"
- }, {
- "name" : "end_lat",
- "type" : "double"
- }, {
- "name" : "end_lon",
- "type" : "double"
- }, {
- "name" : "fare",
- "type" : "double"
- }, {
- "name" : "currency",
- "type" : "string"
- }, {
- "name" : "_hoodie_is_deleted",
- "type" : "boolean",
- "default" : false
- } ]
+ {
+ "name": "timestamp",
+ "type": "long"
+ },
+ {
+ "name": "_row_key",
+ "type": "string"
+ },
+ {
+ "name": "partition_path",
+ "type": "string"
+ },
+ {
+ "name": "trip_type",
+ "type": {
+ "type": "enum",
+ "name": "TripType",
+ "symbols": [
+ "UNKNOWN",
+ "UBERX",
+ "BLACK"
+ ],
+ "default": "UNKNOWN"
+ }
+ },
+ {
+ "name": "rider",
+ "type": "string"
+ },
+ {
+ "name": "driver",
+ "type": "string"
+ },
+ {
+ "name": "begin_lat",
+ "type": "double"
+ },
+ {
+ "name": "begin_lon",
+ "type": "double"
+ },
+ {
+ "name": "end_lat",
+ "type": "double"
+ },
+ {
+ "name": "end_lon",
+ "type": "double"
+ },
+ {
+ "name": "fare",
+ "type": "double"
+ },
+ {
+ "name": "currency",
+ "type": "string"
+ },
+ {
+ "name": "_hoodie_is_deleted",
+ "type": "boolean",
+ "default": false
+ }
+ ]
}
diff --git a/hudi-utilities/src/test/resources/streamer-config/source.avsc b/hudi-utilities/src/test/resources/streamer-config/source.avsc
index a10ab74f92c..510e68de736 100644
--- a/hudi-utilities/src/test/resources/streamer-config/source.avsc
+++ b/hudi-utilities/src/test/resources/streamer-config/source.avsc
@@ -19,114 +19,144 @@
"type" : "record",
"name" : "triprec",
"fields" : [
- {
- "name" : "timestamp",
- "type" : "long"
- }, {
- "name" : "_row_key",
- "type" : "string"
- }, {
- "name" : "partition_path",
- "type" : "string"
- }, {
- "name" : "rider",
- "type" : "string"
- }, {
- "name" : "driver",
- "type" : "string"
- }, {
- "name" : "begin_lat",
- "type" : "double"
- }, {
- "name" : "begin_lon",
- "type" : "double"
- }, {
- "name" : "end_lat",
- "type" : "double"
- }, {
- "name" : "end_lon",
- "type" : "double"
- }, {
- "name" : "distance_in_meters",
- "type" : "int"
- }, {
- "name" : "seconds_since_epoch",
- "type" : "long"
- }, {
- "name" : "weight",
- "type" : "float"
- },{
- "name" : "nation",
- "type" : "bytes"
- },{
- "name" : "current_date",
- "type" : {
- "type" : "int",
- "logicalType" : "date"
+ {
+ "name": "timestamp",
+ "type": "long"
+ },
+ {
+ "name": "_row_key",
+ "type": "string"
+ },
+ {
+ "name": "partition_path",
+ "type": "string"
+ },
+ {
+ "name": "trip_type",
+ "type": {
+ "type": "enum",
+ "name": "TripType",
+ "symbols": [
+ "UNKNOWN",
+ "UBERX",
+ "BLACK"
+ ],
+ "default": "UNKNOWN"
}
- },{
- "name" : "current_ts",
- "type" : {
- "type" : "long"
+ },
+ {
+ "name": "rider",
+ "type": "string"
+ },
+ {
+ "name": "driver",
+ "type": "string"
+ },
+ {
+ "name": "begin_lat",
+ "type": "double"
+ },
+ {
+ "name": "begin_lon",
+ "type": "double"
+ },
+ {
+ "name": "end_lat",
+ "type": "double"
+ },
+ {
+ "name": "end_lon",
+ "type": "double"
+ },
+ {
+ "name": "distance_in_meters",
+ "type": "int"
+ },
+ {
+ "name": "seconds_since_epoch",
+ "type": "long"
+ },
+ {
+ "name": "weight",
+ "type": "float"
+ },
+ {
+ "name": "nation",
+ "type": "bytes"
+ },
+ {
+ "name": "current_date",
+ "type": {
+ "type": "int",
+ "logicalType": "date"
}
- },{
- "name" : "height",
- "type" : {
- "type" : "fixed",
- "name" : "abc",
- "size" : 5,
- "logicalType" : "decimal",
- "precision" : 10,
- "scale": 6
+ },
+ {
+ "name": "current_ts",
+ "type": {
+ "type": "long"
}
- }, {
- "name" :"city_to_state",
- "type" : {
- "type" : "map",
- "values": "string"
- }
- },
- {
- "name" : "fare",
- "type" : {
- "type" : "record",
- "name" : "fare",
- "fields" : [
- {
- "name" : "amount",
- "type" : "double"
- },
- {
- "name" : "currency",
- "type" : "string"
- }
- ]
- }
- },
- {
- "name" : "tip_history",
- "type" : {
- "type" : "array",
- "items" : {
- "type" : "record",
- "name" : "tip_history",
- "fields" : [
+ },
+ {
+ "name": "height",
+ "type": {
+ "type": "fixed",
+ "name": "abc",
+ "size": 5,
+ "logicalType": "decimal",
+ "precision": 10,
+ "scale": 6
+ }
+ },
+ {
+ "name": "city_to_state",
+ "type": {
+ "type": "map",
+ "values": "string"
+ }
+ },
+ {
+ "name": "fare",
+ "type": {
+ "type": "record",
+ "name": "fare",
+ "fields": [
{
- "name" : "amount",
- "type" : "double"
+ "name": "amount",
+ "type": "double"
},
{
- "name" : "currency",
- "type" : "string"
+ "name": "currency",
+ "type": "string"
}
]
}
+ },
+ {
+ "name": "tip_history",
+ "type": {
+ "type": "array",
+ "items": {
+ "type": "record",
+ "name": "tip_history",
+ "fields": [
+ {
+ "name": "amount",
+ "type": "double"
+ },
+ {
+ "name": "currency",
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "name": "_hoodie_is_deleted",
+ "type": "boolean",
+ "default": false
}
- },
- {
- "name" : "_hoodie_is_deleted",
- "type" : "boolean",
- "default" : false
- } ]
+ ]
}
diff --git a/hudi-utilities/src/test/resources/streamer-config/source_evolved.avsc b/hudi-utilities/src/test/resources/streamer-config/source_evolved.avsc
index 4f292647da5..dba040d3525 100644
--- a/hudi-utilities/src/test/resources/streamer-config/source_evolved.avsc
+++ b/hudi-utilities/src/test/resources/streamer-config/source_evolved.avsc
@@ -31,6 +31,10 @@
"name": "partition_path",
"type": "string"
},
+ {
+ "name": "trip_type",
+ "type": "string"
+ },
{
"name": "rider",
"type": "string"
diff --git a/hudi-utilities/src/test/resources/streamer-config/source_evolved_post_processed.avsc b/hudi-utilities/src/test/resources/streamer-config/source_evolved_post_processed.avsc
index 2f820cb920a..af9844218c4 100644
--- a/hudi-utilities/src/test/resources/streamer-config/source_evolved_post_processed.avsc
+++ b/hudi-utilities/src/test/resources/streamer-config/source_evolved_post_processed.avsc
@@ -31,6 +31,10 @@
"name": "partition_path",
"type": "string"
},
+ {
+ "name": "trip_type",
+ "type": "string"
+ },
{
"name": "rider",
"type": "string"
diff --git a/hudi-utilities/src/test/resources/streamer-config/sql-transformer.properties b/hudi-utilities/src/test/resources/streamer-config/sql-transformer.properties
index 93f95d3dc19..9172337d038 100644
--- a/hudi-utilities/src/test/resources/streamer-config/sql-transformer.properties
+++ b/hudi-utilities/src/test/resources/streamer-config/sql-transformer.properties
@@ -16,4 +16,4 @@
# limitations under the License.
###
include=base.properties
-hoodie.deltastreamer.transformer.sql=SELECT a.timestamp, a._row_key, a.partition_path, a.rider, a.driver, a.begin_lat, a.begin_lon, a.end_lat, a.end_lon, a.distance_in_meters, a.seconds_since_epoch, a.weight, a.nation, a.current_date, a.current_ts, a.height, a.city_to_state, a.fare, a.tip_history, a.`_hoodie_is_deleted`, CAST(1.0 AS DOUBLE) AS haversine_distance FROM <SRC> a
+hoodie.deltastreamer.transformer.sql=SELECT a.timestamp, a._row_key, a.partition_path, a.trip_type, a.rider, a.driver, a.begin_lat, a.begin_lon, a.end_lat, a.end_lon, a.distance_in_meters, a.seconds_since_epoch, a.weight, a.nation, a.current_date, a.current_ts, a.height, a.city_to_state, a.fare, a.tip_history, a.`_hoodie_is_deleted`, CAST(1.0 AS DOUBLE) AS haversine_distance FROM <SRC> a
diff --git a/hudi-utilities/src/test/resources/streamer-config/target-flattened.avsc b/hudi-utilities/src/test/resources/streamer-config/target-flattened.avsc
index 2be01709cd9..a61382010ec 100644
--- a/hudi-utilities/src/test/resources/streamer-config/target-flattened.avsc
+++ b/hudi-utilities/src/test/resources/streamer-config/target-flattened.avsc
@@ -19,45 +19,71 @@
"type" : "record",
"name" : "triprec",
"fields" : [
- {
- "name" : "timestamp",
- "type" : "long"
- }, {
- "name" : "_row_key",
- "type" : "string"
- }, {
- "name" : "partition_path",
- "type" : "string"
- }, {
- "name" : "rider",
- "type" : "string"
- }, {
- "name" : "driver",
- "type" : "string"
- }, {
- "name" : "begin_lat",
- "type" : "double"
- }, {
- "name" : "begin_lon",
- "type" : "double"
- }, {
- "name" : "end_lat",
- "type" : "double"
- }, {
- "name" : "end_lon",
- "type" : "double"
- }, {
- "name" : "fare",
- "type" : "double"
- }, {
- "name" : "currency",
- "type" : "string"
- }, {
- "name" : "_hoodie_is_deleted",
- "type" : "boolean",
- "default" : false
- }, {
- "name" : "haversine_distance",
- "type" : "double"
- }]
+ {
+ "name": "timestamp",
+ "type": "long"
+ },
+ {
+ "name": "_row_key",
+ "type": "string"
+ },
+ {
+ "name": "partition_path",
+ "type": "string"
+ },
+ {
+ "name": "trip_type",
+ "type": {
+ "type": "enum",
+ "name": "TripType",
+ "symbols": [
+ "UNKNOWN",
+ "UBERX",
+ "BLACK"
+ ],
+ "default": "UNKNOWN"
+ }
+ },
+ {
+ "name": "rider",
+ "type": "string"
+ },
+ {
+ "name": "driver",
+ "type": "string"
+ },
+ {
+ "name": "begin_lat",
+ "type": "double"
+ },
+ {
+ "name": "begin_lon",
+ "type": "double"
+ },
+ {
+ "name": "end_lat",
+ "type": "double"
+ },
+ {
+ "name": "end_lon",
+ "type": "double"
+ },
+ {
+ "name": "fare",
+ "type": "double"
+ },
+ {
+ "name": "currency",
+ "type": "string"
+ },
+ {
+ "name": "_hoodie_is_deleted",
+ "type": "boolean",
+ "default": false
+ },
+ {
+ "name": "haversine_distance",
+ "type": "double"
+ }
+ ]
}
diff --git a/hudi-utilities/src/test/resources/streamer-config/target.avsc b/hudi-utilities/src/test/resources/streamer-config/target.avsc
index 318dfdcc0be..671be625b59 100644
--- a/hudi-utilities/src/test/resources/streamer-config/target.avsc
+++ b/hudi-utilities/src/test/resources/streamer-config/target.avsc
@@ -19,117 +19,148 @@
"type" : "record",
"name" : "triprec",
"fields" : [
- {
- "name" : "timestamp",
- "type" : "long"
- }, {
- "name" : "_row_key",
- "type" : "string"
- }, {
- "name" : "partition_path",
- "type" : "string"
- }, {
- "name" : "rider",
- "type" : "string"
- }, {
- "name" : "driver",
- "type" : "string"
- }, {
- "name" : "begin_lat",
- "type" : "double"
- }, {
- "name" : "begin_lon",
- "type" : "double"
- }, {
- "name" : "end_lat",
- "type" : "double"
- }, {
- "name" : "end_lon",
- "type" : "double"
- }, {
- "name" : "distance_in_meters",
- "type" : "int"
- }, {
- "name" : "seconds_since_epoch",
- "type" : "long"
- }, {
- "name" : "weight",
- "type" : "float"
- }, {
- "name" : "nation",
- "type" : "bytes"
- },{
- "name" : "current_date",
- "type" : {
- "type" : "int",
- "logicalType" : "date"
+ {
+ "name": "timestamp",
+ "type": "long"
+ },
+ {
+ "name": "_row_key",
+ "type": "string"
+ },
+ {
+ "name": "partition_path",
+ "type": "string"
+ },
+ {
+ "name": "trip_type",
+ "type": {
+ "type": "enum",
+ "name": "TripType",
+ "symbols": [
+ "UNKNOWN",
+ "UBERX",
+ "BLACK"
+ ],
+ "default": "UNKNOWN"
}
- },{
- "name" : "current_ts",
- "type" : {
- "type" : "long"
+ },
+ {
+ "name": "rider",
+ "type": "string"
+ },
+ {
+ "name": "driver",
+ "type": "string"
+ },
+ {
+ "name": "begin_lat",
+ "type": "double"
+ },
+ {
+ "name": "begin_lon",
+ "type": "double"
+ },
+ {
+ "name": "end_lat",
+ "type": "double"
+ },
+ {
+ "name": "end_lon",
+ "type": "double"
+ },
+ {
+ "name": "distance_in_meters",
+ "type": "int"
+ },
+ {
+ "name": "seconds_since_epoch",
+ "type": "long"
+ },
+ {
+ "name": "weight",
+ "type": "float"
+ },
+ {
+ "name": "nation",
+ "type": "bytes"
+ },
+ {
+ "name": "current_date",
+ "type": {
+ "type": "int",
+ "logicalType": "date"
}
- }, {
- "name" : "height",
- "type" : {
- "type" : "fixed",
- "name" : "abc",
- "size" : 5,
- "logicalType" : "decimal",
- "precision" : 10,
- "scale": 6
+ },
+ {
+ "name": "current_ts",
+ "type": {
+ "type": "long"
}
- }, {
- "name" :"city_to_state",
- "type" : {
- "type" : "map",
- "values": "string"
- }
- }, {
- "name" : "fare",
- "type" : {
- "type" : "record",
- "name" : "fare",
- "fields" : [
- {
- "name" : "amount",
- "type" : "double"
- },
- {
- "name" : "currency",
- "type" : "string"
- }
- ]
- }
- },
- {
- "name" : "tip_history",
- "type" : {
- "type" : "array",
- "items" : {
- "type" : "record",
- "name" : "tip_history",
- "fields" : [
+ },
+ {
+ "name": "height",
+ "type": {
+ "type": "fixed",
+ "name": "abc",
+ "size": 5,
+ "logicalType": "decimal",
+ "precision": 10,
+ "scale": 6
+ }
+ },
+ {
+ "name": "city_to_state",
+ "type": {
+ "type": "map",
+ "values": "string"
+ }
+ },
+ {
+ "name": "fare",
+ "type": {
+ "type": "record",
+ "name": "fare",
+ "fields": [
{
- "name" : "amount",
- "type" : "double"
+ "name": "amount",
+ "type": "double"
},
{
- "name" : "currency",
- "type" : "string"
+ "name": "currency",
+ "type": "string"
}
]
}
+ },
+ {
+ "name": "tip_history",
+ "type": {
+ "type": "array",
+ "items": {
+ "type": "record",
+ "name": "tip_history",
+ "fields": [
+ {
+ "name": "amount",
+ "type": "double"
+ },
+ {
+ "name": "currency",
+ "type": "string"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "name": "_hoodie_is_deleted",
+ "type": "boolean",
+ "default": false
+ },
+ {
+ "name": "haversine_distance",
+ "type": "double"
}
- },
- {
- "name" : "_hoodie_is_deleted",
- "type" : "boolean",
- "default" : false
- },
- {
- "name" : "haversine_distance",
- "type" : "double"
- }]
+ ]
}