You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hive.apache.org by "Dharmik Thakkar (Jira)" <ji...@apache.org> on 2023/01/11 08:37:00 UTC
[jira] [Created] (HIVE-26929) Allow creating iceberg tables without column definition when 'metadata_location' tblproperties is set.
Dharmik Thakkar created HIVE-26929:
--------------------------------------
Summary: Allow creating iceberg tables without column definition when 'metadata_location' tblproperties is set.
Key: HIVE-26929
URL: https://issues.apache.org/jira/browse/HIVE-26929
Project: Hive
Issue Type: Improvement
Components: Iceberg integration
Reporter: Dharmik Thakkar
Allow creating iceberg tables without column definition when 'metadata_location' tblproperties is set.
Iceberg supports pointing to external metadata.json file to infer table schema. Irrespective of the schema defined as part of create table statement the metadata.json is used to create table. We should allow creating table without column definition in case the metadata_location is defined in tblproperties.
{code:java}
create table test_meta (id int, name string, cgpa decimal) stored by iceberg stored as orc;
describe formatted test_meta;
create table test_meta_copy(id int) stored by iceberg tblproperties('metadata_location'='s3a://qe-s3-bucket-weekly-dj5h-dwx-external/clusters/env-dqdj5h/warehouse-1673341391-kkzh/warehouse/tablespace/external/hive/iceberg_test_db_hive.db/test_meta/metadata/00000-7dfd7602-f5e1-4473-97cb-79377d358aa3.metadata.json');{code}
As a result of above SQL we get test_meta_copy with same schema as test_meta irrespective of the columns specified in create table statement.
|{color:#000000}*col_name*{color}|{color:#000000}*data_type*{color}|
|{color:#000000}*id*{color}|{color:#000000}int{color}|
|{color:#000000}*name*{color}|{color:#000000}string{color}|
|{color:#000000}*cgpa*{color}|{color:#000000}decimal(10,0){color}|
| |{color:#000000}NULL{color}|
|{color:#000000}*# Detailed Table Information*{color}|{color:#000000}NULL{color}|
|{color:#000000}*Database:* {color}|{color:#000000}iceberg_test_db_hive{color}|
|{color:#000000}*OwnerType: * {color}|{color:#000000}USER {color}|
|{color:#000000}*Owner: * {color}|{color:#000000}hive {color}|
|{color:#000000}*CreateTime:* {color}|{color:#000000}Tue Jan 10 21:49:08 UTC 2023{color}|
|{color:#000000}*LastAccessTime:* {color}|{color:#000000}Fri Dec 12 21:41:41 UTC 1969{color}|
|{color:#000000}*Retention: * {color}|{color:#000000}2147483647{color}|
|{color:#000000}*Location:* {color}|{color:#000000}+s3a://qe-s3-bucket-weekly-dj5h-dwx-external/clusters/env-dqdj5h/warehouse-1673341391-kkzh/warehouse/tablespace/external/hive/iceberg_test_db_hive.db/test_meta+{color}|
|{color:#000000}*Table Type:* {color}|{color:#000000}EXTERNAL_TABLE {color}|
|{color:#000000}*Table Parameters:*{color}|{color:#000000}NULL{color}|
| |{color:#000000}EXTERNAL {color}|
| |{color:#000000}bucketing_version {color}|
| |{color:#000000}engine.hive.enabled{color}|
| |{color:#000000}metadata_location {color}|
| |{color:#000000}numFiles {color}|
| |{color:#000000}numRows {color}|
| |{color:#000000}rawDataSize {color}|
| |{color:#000000}serialization.format{color}|
| |{color:#000000}storage_handler {color}|
| |{color:#000000}table_type {color}|
| |{color:#000000}totalSize {color}|
| |{color:#000000}transient_lastDdlTime{color}|
| |{color:#000000}uuid {color}|
| |{color:#000000}write.format.default{color}|
| |{color:#000000}NULL{color}|
|{color:#000000}*# Storage Information*{color}|{color:#000000}NULL{color}|
|{color:#000000}*SerDe Library: * {color}|{color:#000000}org.apache.iceberg.mr.hive.HiveIcebergSerDe{color}|
|{color:#000000}*InputFormat: * {color}|{color:#000000}org.apache.iceberg.mr.hive.HiveIcebergInputFormat{color}|
|{color:#000000}*OutputFormat:* {color}|{color:#000000}org.apache.iceberg.mr.hive.HiveIcebergOutputFormat{color}|
|{color:#000000}*Compressed:* {color}|{color:#000000}No {color}|
|{color:#000000}*Sort Columns:* {color}|{color:#000000}[] {color}|
However if we skip passing column definition the query fails
{code:java}
create table test_meta_copy2 stored by iceberg tblproperties('metadata_location'='s3a://qe-s3-bucket-weekly-dj5h-dwx-external/clusters/env-dqdj5h/warehouse-1673341391-kkzh/warehouse/tablespace/external/hive/iceberg_test_db_hive.db/test_meta/metadata/00000-7dfd7602-f5e1-4473-97cb-79377d358aa3.metadata.json');{code}
error
{code:java}
INFO : Compiling command(queryId=hive_20230110220019_94ffafef-f531-4532-a07c-0e46e3879f19): create table test_meta_copy2 stored by iceberg tblproperties('metadata_location'='s3a://qe-s3-bucket-weekly-dj5h-dwx-external/clusters/env-dqdj5h/warehouse-1673341391-kkzh/warehouse/tablespace/external/hive/iceberg_test_db_hive.db/test_meta/metadata/00000-7dfd7602-f5e1-4473-97cb-79377d358aa3.metadata.json')
INFO : Semantic Analysis Completed (retrial = false)
INFO : Created Hive schema: Schema(fieldSchemas:null, properties:null)
INFO : Completed compiling command(queryId=hive_20230110220019_94ffafef-f531-4532-a07c-0e46e3879f19); Time taken: 0.019 seconds
INFO : Executing command(queryId=hive_20230110220019_94ffafef-f531-4532-a07c-0e46e3879f19): create table test_meta_copy2 stored by iceberg tblproperties('metadata_location'='s3a://qe-s3-bucket-weekly-dj5h-dwx-external/clusters/env-dqdj5h/warehouse-1673341391-kkzh/warehouse/tablespace/external/hive/iceberg_test_db_hive.db/test_meta/metadata/00000-7dfd7602-f5e1-4473-97cb-79377d358aa3.metadata.json')
INFO : Starting task [Stage-0:DDL] in serial mode
ERROR : Failed
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema)
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1361) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1366) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.createTableNonReplaceMode(CreateTableOperation.java:158) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.execute(CreateTableOperation.java:116) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.ddl.DDLTask.execute(DDLTask.java:84) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:213) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:360) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:333) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:250) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:111) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:809) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:547) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:541) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:166) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:232) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hive.service.cli.operation.SQLOperation.access$700(SQLOperation.java:89) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:338) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at java.security.AccessController.doPrivileged(Native Method) ~[?:?]
at javax.security.auth.Subject.doAs(Subject.java:423) ~[?:?]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899) ~[hadoop-common-3.1.1.7.2.15.4-6.jar:?]
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:358) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:829) [?:?]
Caused by: java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema)
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:349) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:329) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1307) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
... 28 more
Caused by: org.apache.hadoop.hive.metastore.api.MetaException: org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema
at org.apache.hadoop.hive.metastore.HiveMetaStoreUtils.getDeserializer(HiveMetaStoreUtils.java:123) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.metastore.HiveMetaStoreUtils.getDeserializer(HiveMetaStoreUtils.java:80) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:347) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:329) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1307) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
... 28 more
ERROR : DDLTask failed, DDL Operation: class org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema)
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1361) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1366) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.createTableNonReplaceMode(CreateTableOperation.java:158) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation.execute(CreateTableOperation.java:116) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.ddl.DDLTask.execute(DDLTask.java:84) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:213) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:360) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:333) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:250) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:111) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:809) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:547) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:541) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:166) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:232) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hive.service.cli.operation.SQLOperation.access$700(SQLOperation.java:89) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:338) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at java.security.AccessController.doPrivileged(Native Method) ~[?:?]
at javax.security.auth.Subject.doAs(Subject.java:423) ~[?:?]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899) ~[hadoop-common-3.1.1.7.2.15.4-6.jar:?]
at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:358) ~[hive-service-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:829) [?:?]
Caused by: java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema)
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:349) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:329) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1307) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
... 28 more
Caused by: org.apache.hadoop.hive.metastore.api.MetaException: org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema
at org.apache.hadoop.hive.metastore.HiveMetaStoreUtils.getDeserializer(HiveMetaStoreUtils.java:123) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.metastore.HiveMetaStoreUtils.getDeserializer(HiveMetaStoreUtils.java:80) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:347) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:329) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:1307) ~[hive-exec-3.1.3000.2022.0.13.0-72.jar:3.1.3000.2022.0.13.0-72]
... 28 more
ERROR : FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.SerDeException Please provide an existing table or a valid schema)
INFO : Completed executing command(queryId=hive_20230110220019_94ffafef-f531-4532-a07c-0e46e3879f19); Time taken: 0.033 seconds
INFO : OK {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)