You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2022/12/13 11:37:34 UTC

[hudi] 07/17: [HUDI-5294] Support type change for schema on read + reconcile schema (#7326)

This is an automated email from the ASF dual-hosted git repository.

codope pushed a commit to branch release-0.12.2
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 421f1cd6b75b3a9599edd0953273f606ac112e3d
Author: xiarixiaoyao <me...@qq.com>
AuthorDate: Wed Dec 7 10:38:54 2022 +0800

    [HUDI-5294] Support type change for schema on read + reconcile schema (#7326)
    
    * [HUDI-5294] Support type change for schema on read + reconcile schema
---
 .../apache/hudi/client/BaseHoodieWriteClient.java  |   2 +-
 .../hudi/common/util/InternalSchemaCache.java      |  26 ++++-
 .../schema/utils/AvroSchemaEvolutionUtils.java     |  33 ++++---
 .../org/apache/hudi/HoodieSparkSqlWriter.scala     |  32 +++++--
 .../org/apache/spark/sql/hudi/TestSpark3DDL.scala  | 106 ++++++++++++++++++++-
 5 files changed, 172 insertions(+), 27 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index 609f85e27fe..4cec5d1b0c3 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -293,7 +293,7 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K,
       InternalSchema internalSchema;
       Schema avroSchema = HoodieAvroUtils.createHoodieWriteSchema(config.getSchema(), config.allowOperationMetadataField());
       if (historySchemaStr.isEmpty()) {
-        internalSchema = AvroInternalSchemaConverter.convert(avroSchema);
+        internalSchema = SerDeHelper.fromJson(config.getInternalSchema()).orElse(AvroInternalSchemaConverter.convert(avroSchema));
         internalSchema.setSchemaId(Long.parseLong(instantTime));
       } else {
         internalSchema = InternalSchemaUtils.searchSchema(Long.parseLong(instantTime),
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
index 846309b7b67..5f5a8763409 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/InternalSchemaCache.java
@@ -18,6 +18,8 @@
 
 package org.apache.hudi.common.util;
 
+import org.apache.avro.Schema;
+import org.apache.hudi.avro.HoodieAvroUtils;
 import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -26,6 +28,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.internal.schema.InternalSchema;
+import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter;
 import org.apache.hudi.internal.schema.io.FileBasedInternalSchemaStorageManager;
 import org.apache.hudi.internal.schema.utils.InternalSchemaUtils;
 import org.apache.hudi.internal.schema.utils.SerDeHelper;
@@ -166,9 +169,11 @@ public class InternalSchemaCache {
    * step1:
    * try to parser internalSchema from HoodieInstant directly
    * step2:
-   * if we cannot parser internalSchema in step1,
+   * if we cannot parser internalSchema in step1, (eg: current versionId HoodieInstant has been archived)
    * try to find internalSchema in historySchema.
-   *
+   * step3:
+   * if we cannot parser internalSchema in step2  (eg: schema evolution is not enabled when we create hoodie table, however after some inserts we enable schema evolution)
+   * try to convert table schema to internalSchema.
    * @param versionId the internalSchema version to be search.
    * @param tablePath table path
    * @param hadoopConf conf
@@ -176,6 +181,7 @@ public class InternalSchemaCache {
    * @return a internalSchema.
    */
   public static InternalSchema getInternalSchemaByVersionId(long versionId, String tablePath, Configuration hadoopConf, String validCommits) {
+    String avroSchema = "";
     Set<String> commitSet = Arrays.stream(validCommits.split(",")).collect(Collectors.toSet());
     List<String> validateCommitList = commitSet.stream().map(fileName -> {
       String fileExtension = HoodieInstant.getTimelineFileExtension(fileName);
@@ -199,6 +205,7 @@ public class InternalSchemaCache {
         }
         HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(data, HoodieCommitMetadata.class);
         String latestInternalSchemaStr = metadata.getMetadata(SerDeHelper.LATEST_SCHEMA);
+        avroSchema = metadata.getMetadata(HoodieCommitMetadata.SCHEMA_KEY);
         if (latestInternalSchemaStr != null) {
           return SerDeHelper.fromJson(latestInternalSchemaStr).orElse(null);
         }
@@ -209,8 +216,19 @@ public class InternalSchemaCache {
     }
     // step2:
     FileBasedInternalSchemaStorageManager fileBasedInternalSchemaStorageManager = new FileBasedInternalSchemaStorageManager(hadoopConf, new Path(tablePath));
-    String lastestHistorySchema = fileBasedInternalSchemaStorageManager.getHistorySchemaStrByGivenValidCommits(validateCommitList);
-    return InternalSchemaUtils.searchSchema(versionId, SerDeHelper.parseSchemas(lastestHistorySchema));
+    String latestHistorySchema = fileBasedInternalSchemaStorageManager.getHistorySchemaStrByGivenValidCommits(validateCommitList);
+    if (latestHistorySchema.isEmpty()) {
+      return InternalSchema.getEmptyInternalSchema();
+    }
+    InternalSchema fileSchema = InternalSchemaUtils.searchSchema(versionId, SerDeHelper.parseSchemas(latestHistorySchema));
+    // step3:
+    return fileSchema.isEmptySchema() ? AvroInternalSchemaConverter.convert(HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(avroSchema))) : fileSchema;
+  }
+
+  public static InternalSchema getInternalSchemaByVersionId(long versionId, HoodieTableMetaClient metaClient) {
+    String validCommitLists = metaClient
+        .getCommitsAndCompactionTimeline().filterCompletedInstants().getInstantsAsStream().map(HoodieInstant::getFileName).collect(Collectors.joining(","));
+    return getInternalSchemaByVersionId(versionId, metaClient.getBasePathV2().toString(), metaClient.getHadoopConf(), validCommitLists);
   }
 }
 
diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java
index e2b33915853..2561fc507e4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/utils/AvroSchemaEvolutionUtils.java
@@ -19,13 +19,11 @@
 package org.apache.hudi.internal.schema.utils;
 
 import org.apache.hudi.internal.schema.InternalSchema;
-import org.apache.hudi.internal.schema.Types;
 import org.apache.hudi.internal.schema.action.TableChanges;
 import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter;
 
 import org.apache.avro.Schema;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
@@ -41,7 +39,8 @@ public class AvroSchemaEvolutionUtils {
    * 2) incoming data contains new columns not defined yet in the table -> columns will be added to the table schema (incoming dataframe?)
    * 3) incoming data has missing columns that are already defined in the table and new columns not yet defined in the table ->
    *     new columns will be added to the table schema, missing columns will be injected with null values
-   * 4) support nested schema change.
+   * 4) support type change
+   * 5) support nested schema change.
    * Notice:
    *    the incoming schema should not have delete/rename semantics.
    *    for example: incoming schema:  int a, int b, int d;   oldTableSchema int a, int b, int c, int d
@@ -52,25 +51,30 @@ public class AvroSchemaEvolutionUtils {
    */
   public static InternalSchema reconcileSchema(Schema incomingSchema, InternalSchema oldTableSchema) {
     InternalSchema inComingInternalSchema = AvroInternalSchemaConverter.convert(incomingSchema);
-    // do check, only support add column evolution
+    // check column add/missing
     List<String> colNamesFromIncoming = inComingInternalSchema.getAllColsFullName();
     List<String> colNamesFromOldSchema = oldTableSchema.getAllColsFullName();
     List<String> diffFromOldSchema = colNamesFromOldSchema.stream().filter(f -> !colNamesFromIncoming.contains(f)).collect(Collectors.toList());
-    List<Types.Field> newFields = new ArrayList<>();
-    if (colNamesFromIncoming.size() == colNamesFromOldSchema.size() && diffFromOldSchema.size() == 0) {
+    List<String> diffFromEvolutionColumns = colNamesFromIncoming.stream().filter(f -> !colNamesFromOldSchema.contains(f)).collect(Collectors.toList());
+    // check type change.
+    List<String> typeChangeColumns = colNamesFromIncoming
+        .stream()
+        .filter(f -> colNamesFromOldSchema.contains(f) && !inComingInternalSchema.findType(f).equals(oldTableSchema.findType(f)))
+        .collect(Collectors.toList());
+    if (colNamesFromIncoming.size() == colNamesFromOldSchema.size() && diffFromOldSchema.size() == 0 && typeChangeColumns.isEmpty()) {
       return oldTableSchema;
     }
-    List<String> diffFromEvolutionSchema = colNamesFromIncoming.stream().filter(f -> !colNamesFromOldSchema.contains(f)).collect(Collectors.toList());
+
     // Remove redundancy from diffFromEvolutionSchema.
     // for example, now we add a struct col in evolvedSchema, the struct col is " user struct<name:string, age:int> "
     // when we do diff operation: user, user.name, user.age will appeared in the resultSet which is redundancy, user.name and user.age should be excluded.
     // deal with add operation
     TreeMap<Integer, String> finalAddAction = new TreeMap<>();
-    for (int i = 0; i < diffFromEvolutionSchema.size(); i++)  {
-      String name = diffFromEvolutionSchema.get(i);
+    for (int i = 0; i < diffFromEvolutionColumns.size(); i++)  {
+      String name = diffFromEvolutionColumns.get(i);
       int splitPoint = name.lastIndexOf(".");
       String parentName = splitPoint > 0 ? name.substring(0, splitPoint) : "";
-      if (!parentName.isEmpty() && diffFromEvolutionSchema.contains(parentName)) {
+      if (!parentName.isEmpty() && diffFromEvolutionColumns.contains(parentName)) {
         // find redundancy, skip it
         continue;
       }
@@ -94,7 +98,14 @@ public class AvroSchemaEvolutionUtils {
       inferPosition.map(i -> addChange.addPositionChange(name, i, "before"));
     });
 
-    return SchemaChangeUtils.applyTableChanges2Schema(oldTableSchema, addChange);
+    // do type evolution.
+    InternalSchema internalSchemaAfterAddColumns = SchemaChangeUtils.applyTableChanges2Schema(oldTableSchema, addChange);
+    TableChanges.ColumnUpdateChange typeChange = TableChanges.ColumnUpdateChange.get(internalSchemaAfterAddColumns);
+    typeChangeColumns.stream().filter(f -> !inComingInternalSchema.findType(f).isNestedType()).forEach(col -> {
+      typeChange.updateColumnType(col, inComingInternalSchema.findType(col));
+    });
+
+    return SchemaChangeUtils.applyTableChanges2Schema(internalSchemaAfterAddColumns, typeChange);
   }
 
   /**
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 0122576a679..9442195c608 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -258,8 +258,10 @@ object HoodieSparkSqlWriter {
               if (reconcileSchema) {
                 // In case we need to reconcile the schema and schema evolution is enabled,
                 // we will force-apply schema evolution to the writer's schema
-                if (schemaEvolutionEnabled && internalSchemaOpt.isEmpty) {
-                  internalSchemaOpt = Some(AvroInternalSchemaConverter.convert(sourceSchema))
+                if (schemaEvolutionEnabled) {
+                  // in case sourceSchema contains HoodieRecord.HOODIE_META_COLUMNS
+                  val allowOperationMetaDataField = parameters.getOrElse(HoodieWriteConfig.ALLOW_OPERATION_METADATA_FIELD.key(), "false").toBoolean
+                  AvroInternalSchemaConverter.convert(HoodieAvroUtils.addMetadataFields(latestTableSchema, allowOperationMetaDataField))
                 }
 
                 if (internalSchemaOpt.isDefined) {
@@ -314,7 +316,7 @@ object HoodieSparkSqlWriter {
             // Create a HoodieWriteClient & issue the write.
 
             val client = hoodieWriteClient.getOrElse(DataSourceUtils.createHoodieClient(jsc, writerDataSchema.toString, path,
-              tblName, mapAsJavaMap(addSchemaEvolutionParameters(parameters, internalSchemaOpt) - HoodieWriteConfig.AUTO_COMMIT_ENABLE.key)
+              tblName, mapAsJavaMap(addSchemaEvolutionParameters(parameters, internalSchemaOpt, Some(writerDataSchema)) - HoodieWriteConfig.AUTO_COMMIT_ENABLE.key)
             )).asInstanceOf[SparkRDDWriteClient[HoodieRecordPayload[Nothing]]]
 
             if (isAsyncCompactionEnabled(client, tableConfig, parameters, jsc.hadoopConfiguration())) {
@@ -399,10 +401,27 @@ object HoodieSparkSqlWriter {
     processedRecord
   }
 
-  def addSchemaEvolutionParameters(parameters: Map[String, String], internalSchemaOpt: Option[InternalSchema]): Map[String, String] = {
+  def addSchemaEvolutionParameters(parameters: Map[String, String], internalSchemaOpt: Option[InternalSchema], writeSchemaOpt: Option[Schema] = None): Map[String, String] = {
     val schemaEvolutionEnable = if (internalSchemaOpt.isDefined) "true" else "false"
-    parameters ++ Map(HoodieWriteConfig.INTERNAL_SCHEMA_STRING.key() -> SerDeHelper.toJson(internalSchemaOpt.getOrElse(null)),
-      HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key() -> schemaEvolutionEnable)
+
+    val schemaValidateEnable = if (schemaEvolutionEnable.toBoolean && parameters.getOrDefault(DataSourceWriteOptions.RECONCILE_SCHEMA.key(), "false").toBoolean) {
+      // force disable schema validate, now we support schema evolution, no need to do validate
+      "false"
+    } else  {
+      parameters.getOrDefault(HoodieWriteConfig.AVRO_SCHEMA_VALIDATE_ENABLE.key(), "true")
+    }
+    // correct internalSchema, internalSchema should contain hoodie metadata columns.
+    val correctInternalSchema = internalSchemaOpt.map { internalSchema =>
+      if (internalSchema.findField(HoodieRecord.RECORD_KEY_METADATA_FIELD) == null && writeSchemaOpt.isDefined) {
+        val allowOperationMetaDataField = parameters.getOrElse(HoodieWriteConfig.ALLOW_OPERATION_METADATA_FIELD.key(), "false").toBoolean
+        AvroInternalSchemaConverter.convert(HoodieAvroUtils.addMetadataFields(writeSchemaOpt.get, allowOperationMetaDataField))
+      } else {
+        internalSchema
+      }
+    }
+    parameters ++ Map(HoodieWriteConfig.INTERNAL_SCHEMA_STRING.key() -> SerDeHelper.toJson(correctInternalSchema.getOrElse(null)),
+      HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key() -> schemaEvolutionEnable,
+      HoodieWriteConfig.AVRO_SCHEMA_VALIDATE_ENABLE.key()  -> schemaValidateEnable)
   }
 
   /**
@@ -411,7 +430,6 @@ object HoodieSparkSqlWriter {
     * @param fs           instance of FileSystem.
     * @param basePath     base path.
     * @param sparkContext instance of spark context.
-    * @param schema       incoming record's schema.
     * @return Pair of(boolean, table schema), where first entry will be true only if schema conversion is required.
     */
   def getLatestTableInternalSchema(fs: FileSystem, basePath: Path, sparkContext: SparkContext): Option[InternalSchema] = {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
index 9d955cb8310..ed9db3a5aa4 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
@@ -18,13 +18,16 @@
 package org.apache.spark.sql.hudi
 
 import org.apache.hadoop.fs.Path
+import org.apache.hudi.DataSourceWriteOptions.{PARTITIONPATH_FIELD_OPT_KEY, PRECOMBINE_FIELD_OPT_KEY, RECORDKEY_FIELD_OPT_KEY, TABLE_NAME}
+import org.apache.hudi.QuickstartUtils.{DataGenerator, convertToStringList, getQuickstartWriteConfigs}
 import org.apache.hudi.common.model.HoodieRecord
 import org.apache.hudi.common.testutils.HoodieTestDataGenerator
 import org.apache.hudi.common.testutils.RawTripTestPayload
 import org.apache.hudi.config.HoodieWriteConfig
 import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkUtils}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.functions.{arrays_zip, col}
+import org.apache.spark.sql.functions.{arrays_zip, col, expr, lit}
+import org.apache.spark.sql.types.StringType
 import org.apache.spark.sql.{Row, SaveMode, SparkSession}
 
 import scala.collection.JavaConversions._
@@ -511,7 +514,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
     }
   }
 
-  test("Test schema auto evolution") {
+  test("Test schema auto evolution complex") {
     withTempDir { tmp =>
       Seq("COPY_ON_WRITE", "MERGE_ON_READ").foreach { tableType =>
         val tableName = generateTableName
@@ -534,7 +537,6 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
             DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY -> "partition",
             DataSourceWriteOptions.PRECOMBINE_FIELD_OPT_KEY -> "timestamp",
             "hoodie.schema.on.read.enable" -> "true",
-            "hoodie.datasource.write.reconcile.schema" -> "true",
             DataSourceWriteOptions.HIVE_STYLE_PARTITIONING_OPT_KEY -> "true"
           )
 
@@ -546,7 +548,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
             .save(tablePath)
 
           val oldView = spark.read.format("hudi").load(tablePath)
-          oldView.show(false)
+          oldView.show(5, false)
 
           val records2 = RawTripTestPayload.recordsToStrings(dataGen.generateUpdatesAsPerSchema("002", 100, schema)).toList
           val inputD2 = spark.read.json(spark.sparkContext.parallelize(records2, 2))
@@ -571,4 +573,100 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
       }
     }
   }
+
+  test("Test schema auto evolution") {
+    withTempDir { tmp =>
+      Seq("COPY_ON_WRITE", "MERGE_ON_READ").foreach { tableType =>
+        // for complex schema.
+        val tableName = generateTableName
+        val tablePath = s"${new Path(tmp.getCanonicalPath, tableName).toUri.toString}"
+        if (HoodieSparkUtils.gteqSpark3_1) {
+          val dataGen = new DataGenerator
+          val inserts = convertToStringList(dataGen.generateInserts(10))
+          val df = spark.read.json(spark.sparkContext.parallelize(inserts, 2))
+          df.write.format("hudi").
+            options(getQuickstartWriteConfigs).
+            option(DataSourceWriteOptions.TABLE_TYPE_OPT_KEY, tableType).
+            option(PRECOMBINE_FIELD_OPT_KEY, "ts").
+            option(RECORDKEY_FIELD_OPT_KEY, "uuid").
+            option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath").
+            option("hoodie.schema.on.read.enable","true").
+            option(TABLE_NAME.key(), tableName).
+            option("hoodie.table.name", tableName).
+            mode("overwrite").
+            save(tablePath)
+
+          val updates = convertToStringList(dataGen.generateUpdates(10))
+          // type change: fare (double -> String)
+          // add new column and drop a column
+          val dfUpdate = spark.read.json(spark.sparkContext.parallelize(updates, 2))
+            .withColumn("fare", expr("cast(fare as string)"))
+            .withColumn("addColumn", lit("new"))
+          dfUpdate.drop("begin_lat").write.format("hudi").
+            options(getQuickstartWriteConfigs).
+            option(DataSourceWriteOptions.TABLE_TYPE_OPT_KEY, tableType).
+            option(PRECOMBINE_FIELD_OPT_KEY, "ts").
+            option(RECORDKEY_FIELD_OPT_KEY, "uuid").
+            option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath").
+            option("hoodie.schema.on.read.enable","true").
+            option("hoodie.datasource.write.reconcile.schema","true").
+            option(TABLE_NAME.key(), tableName).
+            option("hoodie.table.name", tableName).
+            mode("append").
+            save(tablePath)
+          spark.sql("set hoodie.schema.on.read.enable=true")
+
+          val snapshotDF = spark.read.format("hudi").load(tablePath)
+
+          assertResult(StringType)(snapshotDF.schema.fields.filter(_.name == "fare").head.dataType)
+          assertResult("addColumn")(snapshotDF.schema.fields.last.name)
+          val checkRowKey = dfUpdate.select("fare").collectAsList().map(_.getString(0)).get(0)
+          snapshotDF.createOrReplaceTempView("hudi_trips_snapshot")
+          checkAnswer(spark.sql(s"select fare, addColumn from  hudi_trips_snapshot where fare = ${checkRowKey}").collect())(
+            Seq(checkRowKey, "new")
+          )
+
+          spark.sql(s"select * from  hudi_trips_snapshot").show(false)
+          //  test insert_over_write  + update again
+          val overwrite = convertToStringList(dataGen.generateInserts(10))
+          val dfOverWrite = spark.
+            read.json(spark.sparkContext.parallelize(overwrite, 2)).
+            filter("partitionpath = 'americas/united_states/san_francisco'")
+            .withColumn("fare", expr("cast(fare as string)")) // fare now in table is string type, we forbid convert string to double.
+          dfOverWrite.write.format("hudi").
+            options(getQuickstartWriteConfigs).
+            option("hoodie.datasource.write.operation","insert_overwrite").
+            option(PRECOMBINE_FIELD_OPT_KEY, "ts").
+            option(RECORDKEY_FIELD_OPT_KEY, "uuid").
+            option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath").
+            option("hoodie.schema.on.read.enable","true").
+            option("hoodie.datasource.write.reconcile.schema","true").
+            option(TABLE_NAME.key(), tableName).
+            option("hoodie.table.name", tableName).
+            mode("append").
+            save(tablePath)
+          spark.read.format("hudi").load(tablePath).show(false)
+
+          val updatesAgain = convertToStringList(dataGen.generateUpdates(10))
+          val dfAgain = spark.read.json(spark.sparkContext.parallelize(updatesAgain, 2)).withColumn("fare", expr("cast(fare as string)"))
+          dfAgain.write.format("hudi").
+            options(getQuickstartWriteConfigs).
+            option(PRECOMBINE_FIELD_OPT_KEY, "ts").
+            option(RECORDKEY_FIELD_OPT_KEY, "uuid").
+            option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath").
+            option("hoodie.schema.on.read.enable","true").
+            option("hoodie.datasource.write.reconcile.schema","true").
+            option(TABLE_NAME.key(), tableName).
+            option("hoodie.table.name", tableName).
+            mode("append").
+            save(tablePath)
+          spark.read.format("hudi").load(tablePath).createOrReplaceTempView("hudi_trips_snapshot1")
+          val checkKey = dfAgain.select("fare").collectAsList().map(_.getString(0)).get(0)
+          checkAnswer(spark.sql(s"select fare, addColumn from  hudi_trips_snapshot1 where fare = ${checkKey}").collect())(
+            Seq(checkKey, null)
+          )
+        }
+      }
+    }
+  }
 }