You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues-all@impala.apache.org by "Gabor Kaszab (Jira)" <ji...@apache.org> on 2023/06/13 09:34:00 UTC
[jira] [Updated] (IMPALA-12209) format-version is not present in DESCRIBE FORMATTED and SHOW CREATE TABLE outputs
[ https://issues.apache.org/jira/browse/IMPALA-12209?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Gabor Kaszab updated IMPALA-12209:
----------------------------------
Description:
Repro:
{code:java}
create table tmp (i int, s string) stored as iceberg tblproperties ('format-version'='2');
describe extended/formatted tmp;
show create table tmp;
{code}
Current behaviour:
None of the following 2 commands contain 'format-version' in the output. Additionally, if you run what is returned from SHOW CREATE TABLE then you end up creating a V1 table instead of V2.
The reason might be that format-version in the metadata.json is not stored within the tableproperties but it's on level above:
{code:java}
hdfs dfs -cat hdfs://localhost:20500/test-warehouse/tmp/metadata/00000-55bcfe84-1819-4fb7-ade8-9c132b117880.metadata.json
{
"format-version" : 2,
"table-uuid" : "9f11c0c4-02c7-4688-823c-fe95dbe3ff72",
"location" : "hdfs://localhost:20500/test-warehouse/tmp",
"last-sequence-number" : 0,
"last-updated-ms" : 1686640775184,
"last-column-id" : 2,
"current-schema-id" : 0,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "i",
"required" : false,
"type" : "int"
}, {
"id" : 2,
"name" : "s",
"required" : false,
"type" : "string"
} ]
} ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"engine.hive.enabled" : "true",
"external.table.purge" : "TRUE",
"write.merge.mode" : "merge-on-read",
"write.format.default" : "parquet",
"write.delete.mode" : "merge-on-read",
"OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
"write.update.mode" : "merge-on-read",
"storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"
},
"current-snapshot-id" : -1,
"refs" : { },
"snapshots" : [ ],
"statistics" : [ ],
"snapshot-log" : [ ],
"metadata-log" : [ ]
{code}
was:
Repro:
{code:java}
create table tmp (i int, s string) stored as iceberg tblproperties ('format-version'='2');
describe extended/formatted tmp;
show create table tmp;
{code}
Current behaviour:
Non of the following 2 commands contain 'format-version' in the output. Additionally, if you run what is returned from SHOW CREATE TABLE then you end up creating a V1 table instead of V2.
The reson might be that format-version in the metadata.json is not stored within the tableproperties but it's on level above:
{code:java}
hdfs dfs -cat hdfs://localhost:20500/test-warehouse/tmp/metadata/00000-55bcfe84-1819-4fb7-ade8-9c132b117880.metadata.json
{
"format-version" : 2,
"table-uuid" : "9f11c0c4-02c7-4688-823c-fe95dbe3ff72",
"location" : "hdfs://localhost:20500/test-warehouse/tmp",
"last-sequence-number" : 0,
"last-updated-ms" : 1686640775184,
"last-column-id" : 2,
"current-schema-id" : 0,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "i",
"required" : false,
"type" : "int"
}, {
"id" : 2,
"name" : "s",
"required" : false,
"type" : "string"
} ]
} ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"engine.hive.enabled" : "true",
"external.table.purge" : "TRUE",
"write.merge.mode" : "merge-on-read",
"write.format.default" : "parquet",
"write.delete.mode" : "merge-on-read",
"OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
"write.update.mode" : "merge-on-read",
"storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"
},
"current-snapshot-id" : -1,
"refs" : { },
"snapshots" : [ ],
"statistics" : [ ],
"snapshot-log" : [ ],
"metadata-log" : [ ]
{code}
> format-version is not present in DESCRIBE FORMATTED and SHOW CREATE TABLE outputs
> ---------------------------------------------------------------------------------
>
> Key: IMPALA-12209
> URL: https://issues.apache.org/jira/browse/IMPALA-12209
> Project: IMPALA
> Issue Type: Bug
> Components: from
> Reporter: Gabor Kaszab
> Priority: Major
> Labels: impala-iceberg
>
> Repro:
>
> {code:java}
> create table tmp (i int, s string) stored as iceberg tblproperties ('format-version'='2');
> describe extended/formatted tmp;
> show create table tmp;
> {code}
> Current behaviour:
> None of the following 2 commands contain 'format-version' in the output. Additionally, if you run what is returned from SHOW CREATE TABLE then you end up creating a V1 table instead of V2.
> The reason might be that format-version in the metadata.json is not stored within the tableproperties but it's on level above:
> {code:java}
> hdfs dfs -cat hdfs://localhost:20500/test-warehouse/tmp/metadata/00000-55bcfe84-1819-4fb7-ade8-9c132b117880.metadata.json
> {
> "format-version" : 2,
> "table-uuid" : "9f11c0c4-02c7-4688-823c-fe95dbe3ff72",
> "location" : "hdfs://localhost:20500/test-warehouse/tmp",
> "last-sequence-number" : 0,
> "last-updated-ms" : 1686640775184,
> "last-column-id" : 2,
> "current-schema-id" : 0,
> "schemas" : [ {
> "type" : "struct",
> "schema-id" : 0,
> "fields" : [ {
> "id" : 1,
> "name" : "i",
> "required" : false,
> "type" : "int"
> }, {
> "id" : 2,
> "name" : "s",
> "required" : false,
> "type" : "string"
> } ]
> } ],
> "default-spec-id" : 0,
> "partition-specs" : [ {
> "spec-id" : 0,
> "fields" : [ ]
> } ],
> "last-partition-id" : 999,
> "default-sort-order-id" : 0,
> "sort-orders" : [ {
> "order-id" : 0,
> "fields" : [ ]
> } ],
> "properties" : {
> "engine.hive.enabled" : "true",
> "external.table.purge" : "TRUE",
> "write.merge.mode" : "merge-on-read",
> "write.format.default" : "parquet",
> "write.delete.mode" : "merge-on-read",
> "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
> "write.update.mode" : "merge-on-read",
> "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"
> },
> "current-snapshot-id" : -1,
> "refs" : { },
> "snapshots" : [ ],
> "statistics" : [ ],
> "snapshot-log" : [ ],
> "metadata-log" : [ ]
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-all-unsubscribe@impala.apache.org
For additional commands, e-mail: issues-all-help@impala.apache.org