You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "ganczarek (via GitHub)" <gi...@apache.org> on 2024/02/14 14:57:02 UTC

Re: [I] [SUPPORT] Unsupported schema evolution corrupts Hudi table without error [hudi]

ganczarek commented on issue #8756:
URL: https://github.com/apache/hudi/issues/8756#issuecomment-1943989873

   @ad1happy2go. Thank you for trying to reproduce the problem.
   
   I know this issue is quite old, but I don't have a way to test it with newer versions of Hudi or Hive right now. However, I can still reproduce the problem with Hudi 0.12.2 and Hive 3.1.3 (AWS EMR 6.10.0).
   
   ```
   $ hive --version
   Hive 3.1.3-amzn-3
   Git file:///codebuild/output/src425131955/src/build/hive/rpm/BUILD/apache-hive-3.1.3-amzn-3-src -r Unknown
   Compiled by release on Tue Feb 21 19:18:24 UTC 2023
   From source with checksum 9c4721677b95382c13a0fc6adb9dc41c
   ```
   
   What version of Hive did you use?
   
   The full code with all imports:
   
   ```scala
   import org.apache.hudi.config.HoodieWriteConfig
   import org.apache.hudi.hive.{HiveSyncConfig, NonPartitionedExtractor}
   import org.apache.hudi.keygen.constant.KeyGeneratorOptions
   import org.apache.hudi.sync.common.HoodieSyncConfig
   import org.apache.spark.sql.SaveMode
   
   import java.sql.Timestamp
   import java.time.{LocalDateTime, ZoneOffset}
   
   val hiveUrl = "jdbc:hive2://localhost:10000"
   val s3TablePath = "s3://<redacted>/test_table"
   
   case class TestClass(bool_value: Boolean, timestamp_value: Timestamp)
   case class TestClass2(bool_value: String, timestamp_value: String)
   
   Seq(
     ("1", "a", TestClass(true, Timestamp.from(LocalDateTime.now().toInstant(ZoneOffset.UTC)))),
     ("1", "b", TestClass(false, Timestamp.from(LocalDateTime.now().toInstant(ZoneOffset.UTC))))
   ).toDF("partition", "id", "detail")
     .write
     .format("org.apache.hudi")
     .options(
       Map(
         HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key -> "id",
         KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key -> "id",
         HoodieWriteConfig.TBL_NAME.key -> "test_table",
         HiveSyncConfig.HIVE_SYNC_ENABLED.key -> "true",
         HiveSyncConfig.HIVE_URL.key -> hiveUrl,
         HiveSyncConfig.HIVE_AUTO_CREATE_DATABASE.key -> "true",
         HoodieSyncConfig.META_SYNC_DATABASE_NAME.key -> "test_db",
         HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS.key -> classOf[NonPartitionedExtractor].getName,
         HoodieSyncConfig.META_SYNC_TABLE_NAME.key -> "test_table",
         KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key -> "partition",
         KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key -> "true",
       )
     )
     .mode(SaveMode.Overwrite)
     .save(s3TablePath)
   
   spark.read.format("org.apache.hudi").load(s3TablePath).show(false)
   /**
   +-------------------+---------------------+------------------+----------------------+------------------------------------------------------------------------+---+--------------------------------+---------+
   |_hoodie_commit_time|_hoodie_commit_seqno |_hoodie_record_key|_hoodie_partition_path|_hoodie_file_name                                                       |id |detail                          |partition|
   +-------------------+---------------------+------------------+----------------------+------------------------------------------------------------------------+---+--------------------------------+---------+
   |20240214124653234  |20240214124653234_0_0|a                 |partition=1           |4a12398f-e664-48af-a397-28d5741a0256-0_0-76-74_20240214124653234.parquet|a  |{true, 2024-02-14 12:46:52.992} |1        |
   |20240214124653234  |20240214124653234_0_1|b                 |partition=1           |4a12398f-e664-48af-a397-28d5741a0256-0_0-76-74_20240214124653234.parquet|b  |{false, 2024-02-14 12:46:52.992}|1        |
   +-------------------+---------------------+------------------+----------------------+------------------------------------------------------------------------+---+--------------------------------+---------+
   */
   
   Seq(
     ("2", "c", TestClass2("str1", LocalDateTime.now().toString)),
     ("2", "d", TestClass2("str2", LocalDateTime.now().toString))
   ).toDF("partition", "id", "detail")
     .write
     .format("org.apache.hudi")
     .options(
       Map(
         HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key -> "id",
         KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key -> "id",
         HoodieWriteConfig.TBL_NAME.key -> "test_table",
         HiveSyncConfig.HIVE_SYNC_ENABLED.key -> "true",
         HiveSyncConfig.HIVE_URL.key -> hiveUrl,
         HiveSyncConfig.HIVE_AUTO_CREATE_DATABASE.key -> "true",
         HoodieSyncConfig.META_SYNC_DATABASE_NAME.key -> "test_db",
         HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS.key -> classOf[NonPartitionedExtractor].getName,
         HoodieSyncConfig.META_SYNC_TABLE_NAME.key -> "test_table",
         KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key -> "partition",
         KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key -> "true",
       )
     )
     .mode(SaveMode.Append)
     .save(s3TablePath)
   )
   
   spark.read.format("org.apache.hudi").load(s3TablePath).show(false)
   // java.lang.UnsupportedOperationException: org.apache.parquet.column.values.dictionary.PlainValuesDictionary$PlainLongDictionary
   ```
   
   Table schema in Hive after executing above snippet is:
   ```text
   hive> SHOW CREATE TABLE test_db.test_table;
   OK
   CREATE EXTERNAL TABLE `test_db.test_table`(
     `_hoodie_commit_time` string,
     `_hoodie_commit_seqno` string,
     `_hoodie_record_key` string,
     `_hoodie_partition_path` string,
     `_hoodie_file_name` string,
     `id` string,
     `detail` struct<bool_value:string,timestamp_value:string>)
   PARTITIONED BY (
     `partition` string)
   ROW FORMAT SERDE
     'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
   WITH SERDEPROPERTIES (
     'hoodie.query.as.ro.table'='false',
     'path'='s3://<redacted>/test_table')
   STORED AS INPUTFORMAT
     'org.apache.hudi.hadoop.HoodieParquetInputFormat'
   OUTPUTFORMAT
     'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
   LOCATION
     's3://<redacted>/test_table'
   TBLPROPERTIES (
     'bucketing_version'='2',
     'last_commit_time_sync'='20240214133801211',
     'last_modified_by'='hive',
     'last_modified_time'='1707917907',
     'spark.sql.create.version'='3.3.1-amzn-0',
     'spark.sql.sources.provider'='hudi',
     'spark.sql.sources.schema.numPartCols'='1',
     'spark.sql.sources.schema.numParts'='1',
     'spark.sql.sources.schema.part.0'='{"type":"struct","fields":[{"name":"_hoodie_commit_time","type":"string","nullable":true,"metadata":{}},{"name":"_hoodie_commit_seqno","type":"string","nullable":true,"metadata":{}},{"name":"_hoodie_record_key","type":"string","nullable":true,"metadata":{}},{"name":"_hoodie_partition_path","type":"string","nullable":true,"metadata":{}},{"name":"_hoodie_file_name","type":"string","nullable":true,"metadata":{}},{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"detail","type":{"type":"struct","fields":[{"name":"bool_value","type":"string","nullable":true,"metadata":{}},{"name":"timestamp_value","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"partition","type":"string","nullable":true,"metadata":{}}]}',
     'spark.sql.sources.schema.partCol.0'='partition',
     'transient_lastDdlTime'='1707917907')
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org