You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Lin Liu (Jira)" <ji...@apache.org> on 2025/09/05 21:23:00 UTC
[jira] (HUDI-9791) MDT breaks with hfile reader changes when writing with master using table version 6
[ https://issues.apache.org/jira/browse/HUDI-9791 ]
Lin Liu deleted comment on HUDI-9791:
-------------------------------
was (Author: JIRAUSER301185):
/
{code:java}
➜ ~ spark-3.2Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.2.1 /_/
Using Scala version 2.12.15, OpenJDK 64-Bit Server VM, 1.8.0_392-internalBranch HEADCompiled by user hgao on 2022-01-20T19:26:14ZRevision 4f25b3f71238a00508a356591553f2dfa89f8290Url https://github.com/apache/sparkType --help for more information.➜ ~ export SPARK_VERSION=3.2➜ ~ spark-3.2Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.2.1 /_/
Using Scala version 2.12.15, OpenJDK 64-Bit Server VM, 1.8.0_392-internalBranch HEADCompiled by user hgao on 2022-01-20T19:26:14ZRevision 4f25b3f71238a00508a356591553f2dfa89f8290Url https://github.com/apache/sparkType --help for more information.➜ ~ spark-shell --packages org.apache.hudi:hudi-spark$SPARK_VERSION-bundle_2.12:0.14.1 --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' --conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar':: loading settings :: url = jar:file:/Users/linliu/libraries/spark-3.2.1-bin-hadoop3/jars/ivy-2.5.0.jar!/org/apache/ivy/core/settings/ivysettings.xmlIvy Default Cache set to: /Users/linliu/.ivy2/cacheThe jars for the packages stored in: /Users/linliu/.ivy2/jarsorg.apache.hudi#hudi-spark3.2-bundle_2.12 added as a dependency:: resolving dependencies :: org.apache.spark#spark-submit-parent-4b96494a-df95-4a52-a2ac-322d2d02f5b2;1.0 confs: [default] found org.apache.hudi#hudi-spark3.2-bundle_2.12;0.14.1 in local-m2-cache:: resolution report :: resolve 178ms :: artifacts dl 3ms :: modules in use: org.apache.hudi#hudi-spark3.2-bundle_2.12;0.14.1 from local-m2-cache in [default] --------------------------------------------------------------------- | | modules || artifacts | | conf | number| search|dwnlded|evicted|| number|dwnlded| --------------------------------------------------------------------- | default | 1 | 0 | 0 | 0 || 1 | 0 | ---------------------------------------------------------------------:: retrieving :: org.apache.spark#spark-submit-parent-4b96494a-df95-4a52-a2ac-322d2d02f5b2 confs: [default] 0 artifacts copied, 1 already retrieved (0kB/10ms)25/09/05 14:17:16 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicableUsing Spark's default log4j profile: org/apache/spark/log4j-defaults.propertiesSetting default log level to "WARN".To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).25/09/05 14:17:24 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.Spark context Web UI available at http://lins-mbp.attlocal.net:4041Spark context available as 'sc' (master = local[*], app id = local-1757107044815).Spark session available as 'spark'.Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.2.1 /_/
Using Scala version 2.12.15 (OpenJDK 64-Bit Server VM, Java 1.8.0_392-internal)Type in expressions to have them evaluated.Type :help for more information.
scala> import scala.collection.JavaConversions._import scala.collection.JavaConversions._
scala> import org.apache.spark.sql.SaveMode._import org.apache.spark.sql.SaveMode._
scala> import org.apache.hudi.DataSourceReadOptions._import org.apache.hudi.DataSourceReadOptions._
scala> import org.apache.hudi.DataSourceWriteOptions._import org.apache.hudi.DataSourceWriteOptions._
scala> import org.apache.hudi.common.table.HoodieTableConfig._import org.apache.hudi.common.table.HoodieTableConfig._
scala> import org.apache.hudi.config.HoodieWriteConfig._import org.apache.hudi.config.HoodieWriteConfig._
scala> import org.apache.hudi.keygen.constant.KeyGeneratorOptions._import org.apache.hudi.keygen.constant.KeyGeneratorOptions._
scala> import org.apache.hudi.common.model.HoodieRecordimport org.apache.hudi.common.model.HoodieRecord
scala> import spark.implicits._import spark.implicits._
scala> val tableName = "trips_table"tableName: String = trips_table
scala> val basePath = "file:///tmp/trips_table"basePath: String = file:///tmp/trips_table
scala> val columns = Seq("ts","uuid","rider","driver","fare","city")columns: Seq[String] = List(ts, uuid, rider, driver, fare, city)
scala> val data = | Seq((1695159649087L,"334e26e9-8355-45cc-97c6-c31daf0df330","rider-A","driver-K",19.10,"san_francisco"),<console>:2: error: illegal character '\u00a0' Seq((1695159649087L,"334e26e9-8355-45cc-97c6-c31daf0df330","rider-A","driver-K",19.10,"san_francisco"), ^
scala> (1695091554788L,"e96c4396-3fad-413a-a942-4cb36106d721","rider-C","driver-M",27.70 ,"san_francisco"),<console>:1: error: illegal character '\u00a0' (1695091554788L,"e96c4396-3fad-413a-a942-4cb36106d721","rider-C","driver-M",27.70 ,"san_francisco"), ^<console>:1: error: illegal character '\u00a0' (1695091554788L,"e96c4396-3fad-413a-a942-4cb36106d721","rider-C","driver-M",27.70 ,"san_francisco"), ^
scala> (1695046462179L,"9909a8b1-2d15-4d3d-8ec9-efc48c536a00","rider-D","driver-L",33.90 ,"san_francisco"),<console>:1: error: illegal character '\u00a0' (1695046462179L,"9909a8b1-2d15-4d3d-8ec9-efc48c536a00","rider-D","driver-L",33.90 ,"san_francisco"), ^<console>:1: error: illegal character '\u00a0' (1695046462179L,"9909a8b1-2d15-4d3d-8ec9-efc48c536a00","rider-D","driver-L",33.90 ,"san_francisco"), ^
scala> (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ),<console>:1: error: illegal character '\u00a0' (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ), ^<console>:1: error: illegal character '\u00a0' (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ), ^<console>:1: error: illegal character '\u00a0' (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ), ^<console>:1: error: illegal character '\u00a0' (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ), ^
scala> (1695115999911L,"c8abbe79-8d89-47ea-b4ce-4d224bae5bfa","rider-J","driver-T",17.85,"chennai"));<console>:1: error: illegal character '\u00a0' (1695115999911L,"c8abbe79-8d89-47ea-b4ce-4d224bae5bfa","rider-J","driver-T",17.85,"chennai")); ^<console>:1: error: illegal character '\u00a0' (1695115999911L,"c8abbe79-8d89-47ea-b4ce-4d224bae5bfa","rider-J","driver-T",17.85,"chennai")); ^
scala> var inserts = spark.createDataFrame(data).toDF(columns:_*)<console>:48: error: not found: value data var inserts = spark.createDataFrame(data).toDF(columns:_*) ^
scala> inserts.write.format("hudi"). | option("hoodie.datasource.write.partitionpath.field", "city").<console>:2: error: illegal character '\u00a0' option("hoodie.datasource.write.partitionpath.field", "city"). ^
scala> option("hoodie.table.name", tableName).<console>:1: error: illegal character '\u00a0' option("hoodie.table.name", tableName). ^
scala> option("hoodie.metadata.index.column.stats.enable", "true").<console>:1: error: illegal character '\u00a0' option("hoodie.metadata.index.column.stats.enable", "true"). ^
scala> option("hoodie.datasource.write.table.type", "MERGE_ON_READ").<console>:1: error: illegal character '\u00a0' option("hoodie.datasource.write.table.type", "MERGE_ON_READ"). ^
scala> mode(Overwrite).<console>:1: error: illegal character '\u00a0' mode(Overwrite). ^
scala> save(basePath)<console>:1: error: illegal character '\u00a0' save(basePath) ^
scala> val updatesDf = spark.read.format("hudi").load(basePath).filter($"rider" === "rider-D").withColumn("fare", col("fare") * 10)25/09/05 14:17:54 WARN DFSPropertiesConfiguration: Cannot find HUDI_CONF_DIR, please set it as the dir of hudi-defaults.conf25/09/05 14:17:54 WARN DFSPropertiesConfiguration: Properties file file:/etc/hudi/conf/hudi-defaults.conf not found. Ignoring to load props fileupdatesDf: org.apache.spark.sql.DataFrame = [_hoodie_commit_time: string, _hoodie_commit_seqno: string ... 9 more fields]
scala>
scala> updatesDf.write.format("hudi"). | option("hoodie.datasource.write.operation", "upsert").<console>:2: error: illegal character '\u00a0' option("hoodie.datasource.write.operation", "upsert"). ^
scala> option("hoodie.datasource.write.partitionpath.field", "city").<console>:1: error: illegal character '\u00a0' option("hoodie.datasource.write.partitionpath.field", "city"). ^
scala> option("hoodie.table.name", tableName).<console>:1: error: illegal character '\u00a0' option("hoodie.table.name", tableName). ^
scala> option("hoodie.metadata.index.column.stats.enable", "true").<console>:1: error: illegal character '\u00a0' option("hoodie.metadata.index.column.stats.enable", "true"). ^
scala> option("hoodie.datasource.write.table.type", "MERGE_ON_READ").<console>:1: error: illegal character '\u00a0' option("hoodie.datasource.write.table.type", "MERGE_ON_READ"). ^
scala> mode(Append).<console>:1: error: illegal character '\u00a0' mode(Append). ^
scala> save(basePath)<console>:1: error: illegal character '\u00a0' save(basePath) ^
scala> import scala.collection.JavaConversions._import scala.collection.JavaConversions._
scala> import org.apache.spark.sql.SaveMode._import org.apache.spark.sql.SaveMode._
scala> import org.apache.hudi.DataSourceReadOptions._import org.apache.hudi.DataSourceReadOptions._
scala> import org.apache.hudi.DataSourceWriteOptions._import org.apache.hudi.DataSourceWriteOptions._
scala> import org.apache.hudi.common.table.HoodieTableConfig._import org.apache.hudi.common.table.HoodieTableConfig._
scala> import org.apache.hudi.config.HoodieWriteConfig._import org.apache.hudi.config.HoodieWriteConfig._
scala> import org.apache.hudi.keygen.constant.KeyGeneratorOptions._import org.apache.hudi.keygen.constant.KeyGeneratorOptions._
scala> import org.apache.hudi.common.model.HoodieRecordimport org.apache.hudi.common.model.HoodieRecord
scala> import spark.implicits._import spark.implicits._
scala> val tableName = "trips_table"tableName: String = trips_table
scala> val basePath = "file:///tmp/trips_table"basePath: String = file:///tmp/trips_table
scala> val columns = Seq("ts","uuid","rider","driver","fare","city")columns: Seq[String] = List(ts, uuid, rider, driver, fare, city)
scala> val data = | Seq((1695159649087L,"334e26e9-8355-45cc-97c6-c31daf0df330","rider-A","driver-K",19.10,"san_francisco"), | (1695091554788L,"e96c4396-3fad-413a-a942-4cb36106d721","rider-C","driver-M",27.70 ,"san_francisco"), | (1695046462179L,"9909a8b1-2d15-4d3d-8ec9-efc48c536a00","rider-D","driver-L",33.90 ,"san_francisco"), | (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ), | (1695115999911L,"c8abbe79-8d89-47ea-b4ce-4d224bae5bfa","rider-J","driver-T",17.85,"chennai"));data: Seq[(Long, String, String, String, Double, String)] = List((1695159649087,334e26e9-8355-45cc-97c6-c31daf0df330,rider-A,driver-K,19.1,san_francisco), (1695091554788,e96c4396-3fad-413a-a942-4cb36106d721,rider-C,driver-M,27.7,san_francisco), (1695046462179,9909a8b1-2d15-4d3d-8ec9-efc48c536a00,rider-D,driver-L,33.9,san_francisco), (1695516137016,e3cf430c-889d-4015-bc98-59bdce1e530c,rider-F,driver-P,34.15,sao_paulo), (1695115999911,c8abbe79-8d89-47ea-b4ce-4d224bae5bfa,rider-J,driver-T,17.85,chennai))
scala> var inserts = spark.createDataFrame(data).toDF(columns:_*)inserts: org.apache.spark.sql.DataFrame = [ts: bigint, uuid: string ... 4 more fields]
scala> inserts.write.format("hudi"). | option("hoodie.datasource.write.partitionpath.field", "city"). | option("hoodie.table.name", tableName). | option("hoodie.metadata.index.column.stats.enable", "true"). | option("hoodie.datasource.write.table.type", "MERGE_ON_READ"). | mode(Overwrite). | save(basePath)25/09/05 14:18:44 WARN HoodieSparkSqlWriterInternal: Choosing BULK_INSERT as the operation type since auto record key generation is applicable25/09/05 14:18:44 WARN HoodieSparkSqlWriterInternal: hoodie table at file:/tmp/trips_table already exists. Deleting existing data & overwriting with new data.25/09/05 14:18:45 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist25/09/05 14:18:45 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist25/09/05 14:18:47 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 2.3.025/09/05 14:18:47 WARN ObjectStore: setMetaStoreSchemaVersion called but recording version is disabled: version = 2.3.0, comment = Set by MetaStore linliu@192.168.1.10525/09/05 14:18:48 WARN AutoRecordKeyGenerationUtils$: Precombine field ts will be ignored with auto record key generation enabled25/09/05 14:18:52 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties# WARNING: Unable to get Instrumentation. Dynamic Attach failed. You may add this JAR as -javaagent manually, or supply -Djdk.attach.allowAttachSelf# WARNING: Unable to get Instrumentation. Dynamic Attach failed. You may add this JAR as -javaagent manually, or supply -Djdk.attach.allowAttachSelf# WARNING: Unable to attach Serviceability Agent. Unable to attach even with module exceptions: [org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed.]# WARNING: Unable to attach Serviceability Agent. Unable to attach even with module exceptions: [org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed.]# WARNING: Unable to attach Serviceability Agent. Unable to attach even with module exceptions: [org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed.]
scala> val updatesDf = spark.read.format("hudi").load(basePath).filter($"rider" === "rider-D").withColumn("fare", col("fare") * 10)updatesDf: org.apache.spark.sql.DataFrame = [_hoodie_commit_time: string, _hoodie_commit_seqno: string ... 9 more fields]
scala>
scala> updatesDf.write.format("hudi"). | option("hoodie.datasource.write.operation", "upsert"). | option("hoodie.datasource.write.partitionpath.field", "city"). | option("hoodie.table.name", tableName). | option("hoodie.metadata.index.column.stats.enable", "true"). | option("hoodie.datasource.write.table.type", "MERGE_ON_READ"). | mode(Append). | save(basePath)25/09/05 14:19:04 WARN HoodieWriterUtils$: Changing operation type to UPSERT PREPPED for pk less table upserts25/09/05 14:19:04 WARN AutoRecordKeyGenerationUtils$: Precombine field ts will be ignored with auto record key generation enabled25/09/05 14:19:08 WARN HoodieSparkSqlWriterInternal: Closing write client
scala> exit<console>:73: error: not found: value exit exit ^
scala> %➜ ~ spark-3.5Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.5.0 /_/
Using Scala version 2.12.18, OpenJDK 64-Bit Server VM, 1.8.0_392-internalBranch HEADCompiled by user ubuntu on 2023-09-09T01:53:20ZRevision ce5ddad990373636e94071e7cef2f31021add07bUrl https://github.com/apache/sparkType --help for more information.➜ ~ export SPARK_VERSION=3.5 # or 3.4, 3.3spark-shell --packages org.apache.hudi:hudi-spark$SPARK_VERSION-bundle_2.12:1.1.0-SNAPSHOT \--conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \--conf 'spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog' \--conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' \--conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar':: loading settings :: url = jar:file:/Users/linliu/libraries/spark-3.5.0-bin-hadoop3/jars/ivy-2.5.1.jar!/org/apache/ivy/core/settings/ivysettings.xmlIvy Default Cache set to: /Users/linliu/.ivy2/cacheThe jars for the packages stored in: /Users/linliu/.ivy2/jarsorg.apache.hudi#hudi-spark3.5-bundle_2.12 added as a dependency:: resolving dependencies :: org.apache.spark#spark-submit-parent-98cac2cf-863b-4747-99fd-31654bd46720;1.0 confs: [default] found org.apache.hudi#hudi-spark3.5-bundle_2.12;1.1.0-SNAPSHOT in local-m2-cache found org.apache.hive#hive-storage-api;2.8.1 in local-m2-cache found org.slf4j#slf4j-api;1.7.36 in local-m2-cache:: resolution report :: resolve 290ms :: artifacts dl 15ms :: modules in use: org.apache.hive#hive-storage-api;2.8.1 from local-m2-cache in [default] org.apache.hudi#hudi-spark3.5-bundle_2.12;1.1.0-SNAPSHOT from local-m2-cache in [default] org.slf4j#slf4j-api;1.7.36 from local-m2-cache in [default] --------------------------------------------------------------------- | | modules || artifacts | | conf | number| search|dwnlded|evicted|| number|dwnlded| --------------------------------------------------------------------- | default | 3 | 0 | 0 | 0 || 3 | 0 | ---------------------------------------------------------------------:: retrieving :: org.apache.spark#spark-submit-parent-98cac2cf-863b-4747-99fd-31654bd46720 confs: [default] 0 artifacts copied, 3 already retrieved (0kB/7ms)25/09/05 14:19:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicableSetting default log level to "WARN".To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).25/09/05 14:19:46 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.Spark context Web UI available at http://lins-mbp.attlocal.net:4041Spark context available as 'sc' (master = local[*], app id = local-1757107186666).Spark session available as 'spark'.Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.5.0 /_/
Using Scala version 2.12.18 (OpenJDK 64-Bit Server VM, Java 1.8.0_392-internal)Type in expressions to have them evaluated.Type :help for more information.
scala> import scala.collection.JavaConversions._import scala.collection.JavaConversions._
scala> import org.apache.spark.sql.SaveMode._import org.apache.spark.sql.SaveMode._
scala> import org.apache.hudi.DataSourceReadOptions._import org.apache.hudi.DataSourceReadOptions._
scala> import org.apache.hudi.DataSourceWriteOptions._import org.apache.hudi.DataSourceWriteOptions._
scala> import org.apache.hudi.common.table.HoodieTableConfig._import org.apache.hudi.common.table.HoodieTableConfig._
scala> import org.apache.hudi.config.HoodieWriteConfig._import org.apache.hudi.config.HoodieWriteConfig._
scala> import org.apache.hudi.keygen.constant.KeyGeneratorOptions._import org.apache.hudi.keygen.constant.KeyGeneratorOptions._
scala> import org.apache.hudi.common.model.HoodieRecordimport org.apache.hudi.common.model.HoodieRecord
scala> import spark.implicits._import spark.implicits._
scala> val tableName = "trips_table"tableName: String = trips_table
scala> val basePath = "file:///tmp/trips_table"basePath: String = file:///tmp/trips_table
scala> val updatesDf = spark.read.format("hudi").load(basePath).filter($"rider" === "rider-D").withColumn("fare", col("fare") * 10)25/09/05 14:20:12 WARN DFSPropertiesConfiguration: Properties file file:/etc/hudi/conf/hudi-defaults.conf not found. Ignoring to load props file25/09/05 14:20:12 WARN DFSPropertiesConfiguration: Cannot find HUDI_CONF_DIR, please set it as the dir of hudi-defaults.conf25/09/05 14:20:13 WARN ConfigUtils: The configuration key 'hoodie.compaction.record.merger.strategy' has been deprecated and may be removed in the future. Please use the new key 'hoodie.record.merge.strategy.id' instead.updatesDf: org.apache.spark.sql.DataFrame = [_hoodie_commit_time: string, _hoodie_commit_seqno: string ... 9 more fields]
scala>
scala> updatesDf.write.format("hudi").option("hoodie.datasource.write.operation", "upsert").option("hoodie.datasource.write.partitionpath.field", "city").option("hoodie.table.name", tableName).option("hoodie.metadata.index.column.stats.enable", "true").option("hoodie.write.table.version", "6").option("hoodie.datasource.write.table.type", "MERGE_ON_READ").mode(Append).save(basePath)# WARNING: Unable to attach Serviceability Agent. Unable to attach even with module exceptions: [org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed.]25/09/05 14:20:16 WARN ConfigUtils: The configuration key 'hoodie.compaction.record.merger.strategy' has been deprecated and may be removed in the future. Please use the new key 'hoodie.record.merge.strategy.id' instead.25/09/05 14:20:16 WARN ConfigUtils: The configuration key 'hoodie.compaction.record.merger.strategy' has been deprecated and may be removed in the future. Please use the new key 'hoodie.record.merge.strategy.id' instead.25/09/05 14:20:16 WARN HoodieWriterUtils$: Changing operation type to UPSERT PREPPED for pk less table upserts25/09/05 14:20:16 WARN AutoRecordKeyGenerationUtils$: Precombine field ts will be ignored with auto record key generation enabled25/09/05 14:20:16 WARN HoodieWriteConfig: HoodieTableVersion.SIX is not yet fully supported by the writer. Please expect some unexpected behavior, until its fully implemented.25/09/05 14:20:22 WARN HoodieWriteConfig: HoodieTableVersion.SIX is not yet fully supported by the writer. Please expect some unexpected behavior, until its fully implemented.25/09/05 14:20:23 WARN ConfigUtils: The configuration key 'hoodie.compaction.record.merger.strategy' has been deprecated and may be removed in the future. Please use the new key 'hoodie.record.merge.strategy.id' instead.25/09/05 14:20:27 WARN HoodieWriteConfig: HoodieTableVersion.SIX is not yet fully supported by the writer. Please expect some unexpected behavior, until its fully implemented.25/09/05 14:20:28 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
scala> %➜ ~ spark-3.2Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.2.1 /_/
Using Scala version 2.12.15, OpenJDK 64-Bit Server VM, 1.8.0_392-internalBranch HEADCompiled by user hgao on 2022-01-20T19:26:14ZRevision 4f25b3f71238a00508a356591553f2dfa89f8290Url https://github.com/apache/sparkType --help for more information.➜ ~ export SPARK_VERSION=3.2➜ ~ spark-shell --packages org.apache.hudi:hudi-spark$SPARK_VERSION-bundle_2.12:0.14.1 --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' --conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar':: loading settings :: url = jar:file:/Users/linliu/libraries/spark-3.2.1-bin-hadoop3/jars/ivy-2.5.0.jar!/org/apache/ivy/core/settings/ivysettings.xmlIvy Default Cache set to: /Users/linliu/.ivy2/cacheThe jars for the packages stored in: /Users/linliu/.ivy2/jarsorg.apache.hudi#hudi-spark3.2-bundle_2.12 added as a dependency:: resolving dependencies :: org.apache.spark#spark-submit-parent-79f360af-ce55-47c6-a9d1-9bcbe74c0b03;1.0 confs: [default] found org.apache.hudi#hudi-spark3.2-bundle_2.12;0.14.1 in local-m2-cache:: resolution report :: resolve 180ms :: artifacts dl 4ms :: modules in use: org.apache.hudi#hudi-spark3.2-bundle_2.12;0.14.1 from local-m2-cache in [default] --------------------------------------------------------------------- | | modules || artifacts | | conf | number| search|dwnlded|evicted|| number|dwnlded| --------------------------------------------------------------------- | default | 1 | 0 | 0 | 0 || 1 | 0 | ---------------------------------------------------------------------:: retrieving :: org.apache.spark#spark-submit-parent-79f360af-ce55-47c6-a9d1-9bcbe74c0b03 confs: [default] 0 artifacts copied, 1 already retrieved (0kB/6ms)25/09/05 14:21:01 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicableUsing Spark's default log4j profile: org/apache/spark/log4j-defaults.propertiesSetting default log level to "WARN".To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).25/09/05 14:21:09 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.Spark context Web UI available at http://lins-mbp.attlocal.net:4041Spark context available as 'sc' (master = local[*], app id = local-1757107269339).Spark session available as 'spark'.Welcome to ____ __ / __/__ ___ _____/ /__ _\ \/ _ \/ _ `/ __/ '_/ /___/ .__/\_,_/_/ /_/\_\ version 3.2.1 /_/
Using Scala version 2.12.15 (OpenJDK 64-Bit Server VM, Java 1.8.0_392-internal)Type in expressions to have them evaluated.Type :help for more information.
scala> spark.read.format("hudi").option("hoodie.metadata.enable", "true").option("hoodie.enable.data.skipping", "true").option("hoodie.metadata.index.column.stats.enable", "true").load("/tmp/trips_table").filter("fare > 100").show(100,false)25/09/05 14:21:18 WARN DFSPropertiesConfiguration: Cannot find HUDI_CONF_DIR, please set it as the dir of hudi-defaults.conf25/09/05 14:21:18 WARN DFSPropertiesConfiguration: Properties file file:/etc/hudi/conf/hudi-defaults.conf not found. Ignoring to load props file# WARNING: Unable to attach Serviceability Agent. Unable to attach even with module exceptions: [org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed., org.apache.hudi.org.openjdk.jol.vm.sa.SASupportException: Sense failed.]25/09/05 14:21:20 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties+-------------------+---------------------+---------------------+----------------------+--------------------------------------+-------------+------------------------------------+-------+--------+------+-------------+|_hoodie_commit_time|_hoodie_commit_seqno |_hoodie_record_key |_hoodie_partition_path|_hoodie_file_name |ts |uuid |rider |driver |fare |city |+-------------------+---------------------+---------------------+----------------------+--------------------------------------+-------------+------------------------------------+-------+--------+------+-------------+|20250905142017073 |20250905142017073_0_1|20250905141844928_2_0|san_francisco |fbe58388-7307-4159-90fa-f2e0b63f111d-0|1695046462179|9909a8b1-2d15-4d3d-8ec9-efc48c536a00|rider-D|driver-L|3390.0|san_francisco|+-------------------+---------------------+---------------------+----------------------+--------------------------------------+-------------+------------------------------------+-------+--------+------+-------------+
{code}
> MDT breaks with hfile reader changes when writing with master using table version 6
> -----------------------------------------------------------------------------------
>
> Key: HUDI-9791
> URL: https://issues.apache.org/jira/browse/HUDI-9791
> Project: Apache Hudi
> Issue Type: Bug
> Components: metadata
> Reporter: Jonathan Vexler
> Assignee: Lin Liu
> Priority: Blocker
> Fix For: 1.1.0
>
> Attachments: hfile.error
>
>
> Exception: [^hfile.error]
> Summary:
> # Write an insert and update to mor table with spark 3.1 hudi 0.14.1
> # write an update with spark 3.5 with current master `adda6950e0aaa7353add88ee2fc0499d7135ee33` using write table version 6
> # Read table with spark 3.1 hudi 0.14.1 and get exception
> The hoodie.properties still says table version is 6
> Here is my runscript:
> {code:java}
> set_spark 3.1
> hudi_spark_shell -p -v 0.14.1
> import scala.collection.JavaConversions._
> import org.apache.spark.sql.SaveMode._
> import org.apache.hudi.DataSourceReadOptions._
> import org.apache.hudi.DataSourceWriteOptions._
> import org.apache.hudi.common.table.HoodieTableConfig._
> import org.apache.hudi.config.HoodieWriteConfig._
> import org.apache.hudi.keygen.constant.KeyGeneratorOptions._
> import org.apache.hudi.common.model.HoodieRecord
> import spark.implicits._
> val tableName = "trips_table"
> val basePath = "file:///tmp/trips_table"
> val columns = Seq("ts","uuid","rider","driver","fare","city")
> val data =
> Seq((1695159649087L,"334e26e9-8355-45cc-97c6-c31daf0df330","rider-A","driver-K",19.10,"san_francisco"),
> (1695091554788L,"e96c4396-3fad-413a-a942-4cb36106d721","rider-C","driver-M",27.70 ,"san_francisco"),
> (1695046462179L,"9909a8b1-2d15-4d3d-8ec9-efc48c536a00","rider-D","driver-L",33.90 ,"san_francisco"),
> (1695516137016L,"e3cf430c-889d-4015-bc98-59bdce1e530c","rider-F","driver-P",34.15,"sao_paulo" ),
> (1695115999911L,"c8abbe79-8d89-47ea-b4ce-4d224bae5bfa","rider-J","driver-T",17.85,"chennai"));
> var inserts = spark.createDataFrame(data).toDF(columns:_*)
> inserts.write.format("hudi").
> option("hoodie.datasource.write.partitionpath.field", "city").
> option("hoodie.table.name", tableName).
> option("hoodie.metadata.index.column.stats.enable", "true").
> option("hoodie.datasource.write.table.type", "MERGE_ON_READ").
> mode(Overwrite).
> save(basePath)
> val updatesDf = spark.read.format("hudi").load(basePath).filter($"rider" === "rider-D").withColumn("fare", col("fare") * 10)
> updatesDf.write.format("hudi").
> option("hoodie.datasource.write.operation", "upsert").
> option("hoodie.datasource.write.partitionpath.field", "city").
> option("hoodie.table.name", tableName).
> option("hoodie.metadata.index.column.stats.enable", "true").
> option("hoodie.datasource.write.table.type", "MERGE_ON_READ").
> mode(Append).
> save(basePath)
> //exit
> set_spark 3.5
> hudi_spark_shell -j
> import scala.collection.JavaConversions._
> import org.apache.spark.sql.SaveMode._
> import org.apache.hudi.DataSourceReadOptions._
> import org.apache.hudi.DataSourceWriteOptions._
> import org.apache.hudi.common.table.HoodieTableConfig._
> import org.apache.hudi.config.HoodieWriteConfig._
> import org.apache.hudi.keygen.constant.KeyGeneratorOptions._
> import org.apache.hudi.common.model.HoodieRecord
> import spark.implicits._
> val tableName = "trips_table"
> val basePath = "file:///tmp/trips_table"
> val updatesDf = spark.read.format("hudi").load(basePath).filter($"rider" === "rider-D").withColumn("fare", col("fare") * 10)
> updatesDf.write.format("hudi").option("hoodie.datasource.write.operation", "upsert").option("hoodie.datasource.write.partitionpath.field", "city").option("hoodie.table.name", tableName).option("hoodie.metadata.index.column.stats.enable", "true").option("hoodie.write.table.version", "6").option("hoodie.datasource.write.table.type", "MERGE_ON_READ").mode(Append).save(basePath)
> //exit
> set_spark 3.1
> hudi_spark_shell -p -v 0.14.1
> spark.read.format("hudi").option("hoodie.metadata.enable", "true").option("hoodie.enable.data.skipping", "true").option("hoodie.metadata.index.column.stats.enable", "true").load("/tmp/trips_table").filter("fare > 100").show(100,false) {code}
> Command for running 0.14.1 with spark 3.1 using mvn package:
> {code:java}
> /Users/jon/Documents/sparkroot/spark-3.1.3-bin-hadoop3.2/bin/spark-shell --packages org.apache.hudi:hudi-spark3.1-bundle_2.12:0.14.1 --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' --conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar' --conf 'spark.sql.catalogImplementation=in-memory' {code}
> Command for running with current master on spark 3.5
> {code:java}
> /Users/jon/Documents/sparkroot/spark-3.5.2-bin-hadoop3/bin/spark-shell --jars /Users/jon/git/hudi-versions/current/spark3.5/hudi/packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-1.1.0-SNAPSHOT.jar --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' --conf 'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar' --conf 'spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog' --conf 'spark.sql.catalogImplementation=in-memory' {code}
>
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)