You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2019/09/12 13:55:20 UTC
[incubator-iceberg] branch master updated: Spark: Add a unit test
for importing existing tables as Iceberg Hive tables (#461)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new adfa2a1 Spark: Add a unit test for importing existing tables as Iceberg Hive tables (#461)
adfa2a1 is described below
commit adfa2a192c7b6cc2cfcab53bebcacee3357aeb45
Author: Chen, Junjie <ji...@tencent.com>
AuthorDate: Thu Sep 12 21:55:15 2019 +0800
Spark: Add a unit test for importing existing tables as Iceberg Hive tables (#461)
---
.../iceberg/spark/source/TestSparkTableUtil.java | 26 ++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java
index 8c0235f..8cd3af6 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java
@@ -162,4 +162,30 @@ public class TestSparkTableUtil extends HiveTableBaseTest {
long count = spark.read().format("iceberg").load(location.toString()).count();
Assert.assertEquals("three values ", 3, count);
}
+
+ @Test
+ public void testImportAsHiveTable() throws Exception {
+ spark.table(qualifiedTableName).write().mode("overwrite").format("parquet")
+ .saveAsTable("unpartitioned_table");
+ TableIdentifier source = new TableIdentifier("unpartitioned_table");
+ Table table = catalog.createTable(
+ org.apache.iceberg.catalog.TableIdentifier.of(DB_NAME, "test_unpartitioned_table"),
+ SparkSchemaUtil.schemaForTable(spark, qualifiedTableName),
+ SparkSchemaUtil.specForTable(spark, qualifiedTableName));
+ SparkTableUtil.importSparkTable(source, "/tmp", table);
+ long count1 = spark.read().format("iceberg").load(DB_NAME + ".test_unpartitioned_table").count();
+ Assert.assertEquals("three values ", 3, count1);
+
+ spark.table(qualifiedTableName).write().mode("overwrite").partitionBy("data").format("parquet")
+ .saveAsTable("partitioned_table");
+ source = new TableIdentifier("partitioned_table");
+ table = catalog.createTable(
+ org.apache.iceberg.catalog.TableIdentifier.of(DB_NAME, "test_partitioned_table"),
+ SparkSchemaUtil.schemaForTable(spark, qualifiedTableName),
+ SparkSchemaUtil.specForTable(spark, qualifiedTableName));
+
+ SparkTableUtil.importSparkTable(source, "/tmp", table);
+ long count2 = spark.read().format("iceberg").load(DB_NAME + ".test_partitioned_table").count();
+ Assert.assertEquals("three values ", 3, count2);
+ }
}