You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/10/10 22:20:54 UTC
[iceberg] branch master updated: Hive: Run storage handler tests
for Parquet and ORC (#1570)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 60ebbf0 Hive: Run storage handler tests for Parquet and ORC (#1570)
60ebbf0 is described below
commit 60ebbf0875bfc8be698faf68c4f9de3fef635058
Author: Marton Bod <ma...@gmail.com>
AuthorDate: Sun Oct 11 00:20:41 2020 +0200
Hive: Run storage handler tests for Parquet and ORC (#1570)
Co-authored-by: Marton Bod <mb...@cloudera.com>
---
.../mr/hive/HiveIcebergStorageHandlerBaseTest.java | 57 +++++++++++++++++-----
1 file changed, 44 insertions(+), 13 deletions(-)
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
index 857e0db..5bab8ac 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandlerBaseTest.java
@@ -136,18 +136,50 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
}
}
+ // PARQUET
+
+ @Test
+ public void testScanEmptyTableParquet() throws IOException {
+ testScanEmptyTable(FileFormat.PARQUET);
+ }
+
+ @Test
+ public void testScanTableParquet() throws IOException {
+ testScanTable(FileFormat.PARQUET);
+ }
+
+ @Test
+ public void testJoinTablesParquet() throws IOException {
+ testJoinTables(FileFormat.PARQUET);
+ }
+
+ // ORC
+
@Test
- public void testScanEmptyTable() throws IOException {
+ public void testScanEmptyTableORC() throws IOException {
+ testScanEmptyTable(FileFormat.ORC);
+ }
+
+ @Test
+ public void testScanTableORC() throws IOException {
+ testScanTable(FileFormat.ORC);
+ }
+
+ @Test
+ public void testJoinTablesORC() throws IOException {
+ testJoinTables(FileFormat.ORC);
+ }
+
+ public void testScanEmptyTable(FileFormat format) throws IOException {
Schema emptySchema = new Schema(required(1, "empty", Types.StringType.get()));
- createTable("empty", emptySchema, ImmutableList.of());
+ createTable("empty", emptySchema, format, ImmutableList.of());
List<Object[]> rows = shell.executeStatement("SELECT * FROM default.empty");
Assert.assertEquals(0, rows.size());
}
- @Test
- public void testScanTable() throws IOException {
- createTable("customers", CUSTOMER_SCHEMA, CUSTOMER_RECORDS);
+ public void testScanTable(FileFormat format) throws IOException {
+ createTable("customers", CUSTOMER_SCHEMA, format, CUSTOMER_RECORDS);
// Single fetch task: no MR job.
List<Object[]> rows = shell.executeStatement("SELECT * FROM default.customers");
@@ -166,10 +198,9 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
Assert.assertArrayEquals(new Object[] {0L, "Alice"}, descRows.get(2));
}
- @Test
- public void testJoinTables() throws IOException {
- createTable("customers", CUSTOMER_SCHEMA, CUSTOMER_RECORDS);
- createTable("orders", ORDER_SCHEMA, ORDER_RECORDS);
+ public void testJoinTables(FileFormat format) throws IOException {
+ createTable("customers", CUSTOMER_SCHEMA, format, CUSTOMER_RECORDS);
+ createTable("orders", ORDER_SCHEMA, format, ORDER_RECORDS);
List<Object[]> rows = shell.executeStatement(
"SELECT c.customer_id, c.first_name, o.order_id, o.total " +
@@ -182,17 +213,17 @@ public abstract class HiveIcebergStorageHandlerBaseTest {
Assert.assertArrayEquals(new Object[] {1L, "Bob", 102L, 33.33d}, rows.get(2));
}
- protected void createTable(String tableName, Schema schema, List<Record> records)
+ protected void createTable(String tableName, Schema schema, FileFormat format, List<Record> records)
throws IOException {
- Table table = createIcebergTable(tableName, schema, records);
+ Table table = createIcebergTable(tableName, schema, format, records);
createHiveTable(tableName, table.location());
}
- protected Table createIcebergTable(String tableName, Schema schema, List<Record> records)
+ protected Table createIcebergTable(String tableName, Schema schema, FileFormat format, List<Record> records)
throws IOException {
String identifier = testTables.identifier("default." + tableName);
TestHelper helper = new TestHelper(
- metastore.hiveConf(), testTables.tables(), identifier, schema, SPEC, FileFormat.PARQUET, temp);
+ metastore.hiveConf(), testTables.tables(), identifier, schema, SPEC, format, temp);
Table table = helper.createTable();
if (!records.isEmpty()) {