You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2021/03/24 18:41:18 UTC
[iceberg] 03/18: Hive: Fix file extensions for written files (#2155)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch 0.11.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 1554cd664e7e633a3ceed2968e3e280d8b1c4fcb
Author: pvary <pv...@cloudera.com>
AuthorDate: Wed Jan 27 09:08:15 2021 +0100
Hive: Fix file extensions for written files (#2155)
---
.../main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputFormat.java | 2 +-
.../iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputFormat.java
index 95b6b69..6ebc677 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputFormat.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputFormat.java
@@ -74,7 +74,7 @@ public class HiveIcebergOutputFormat<T> implements OutputFormat<NullWritable, Co
LocationProvider location = HiveIcebergStorageHandler.location(jc);
EncryptionManager encryption = HiveIcebergStorageHandler.encryption(jc);
OutputFileFactory outputFileFactory =
- new OutputFileFactory(spec, FileFormat.PARQUET, location, io, encryption, taskAttemptID.getTaskID().getId(),
+ new OutputFileFactory(spec, fileFormat, location, io, encryption, taskAttemptID.getTaskID().getId(),
taskAttemptID.getId(), jc.get(HiveConf.ConfVars.HIVEQUERYID.varname) + "-" + taskAttemptID.getJobID());
HiveIcebergRecordWriter writer = new HiveIcebergRecordWriter(schema, spec, fileFormat,
new GenericAppenderFactory(schema), outputFileFactory, io, targetFileSize, taskAttemptID);
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
index 78f65d6..7200e3d 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
@@ -34,6 +34,7 @@ import org.apache.iceberg.PartitionSpecParser;
import org.apache.iceberg.Schema;
import org.apache.iceberg.SchemaParser;
import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
@@ -535,7 +536,7 @@ public class TestHiveIcebergStorageHandlerWithEngine {
SchemaParser.toJson(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) + "', " +
"'" + InputFormatConfig.PARTITION_SPEC + "'='" +
PartitionSpecParser.toJson(spec) + "', " +
- "'" + InputFormatConfig.WRITE_FILE_FORMAT + "'='" + fileFormat + "')");
+ "'" + TableProperties.DEFAULT_FILE_FORMAT + "'='" + fileFormat + "')");
List<Record> records = TestHelper.generateRandomRecords(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, 4, 0L);