You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/11/01 19:47:23 UTC
[iceberg] 03/06: Hive: Fix Catalogs.hiveCatalog method for default
catalogs (#3338)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch 0.12.x
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 94103eaabe556f04103ba2f597c5868289a354cc
Author: pvary <pv...@cloudera.com>
AuthorDate: Wed Oct 27 00:21:43 2021 +0200
Hive: Fix Catalogs.hiveCatalog method for default catalogs (#3338)
---
.../main/java/org/apache/iceberg/mr/Catalogs.java | 14 +++++++++----
.../java/org/apache/iceberg/mr/TestCatalogs.java | 23 +++++++++++++++++++++-
2 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/mr/src/main/java/org/apache/iceberg/mr/Catalogs.java b/mr/src/main/java/org/apache/iceberg/mr/Catalogs.java
index 2e54cf5..0bf0473 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/Catalogs.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/Catalogs.java
@@ -202,7 +202,15 @@ public final class Catalogs {
*/
public static boolean hiveCatalog(Configuration conf, Properties props) {
String catalogName = props.getProperty(InputFormatConfig.CATALOG_NAME);
- return CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE.equalsIgnoreCase(getCatalogType(conf, catalogName));
+ String catalogType = getCatalogType(conf, catalogName);
+ if (catalogType != null) {
+ return CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE.equalsIgnoreCase(catalogType);
+ }
+ catalogType = getCatalogType(conf, ICEBERG_DEFAULT_CATALOG_NAME);
+ if (catalogType != null) {
+ return CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE.equalsIgnoreCase(catalogType);
+ }
+ return getCatalogProperties(conf, catalogName, catalogType).get(CatalogProperties.CATALOG_IMPL) == null;
}
@VisibleForTesting
@@ -279,9 +287,7 @@ public final class Catalogs {
}
} else {
String catalogType = conf.get(InputFormatConfig.CATALOG);
- if (catalogType == null) {
- return CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE;
- } else if (catalogType.equals(LOCATION)) {
+ if (catalogType != null && catalogType.equals(LOCATION)) {
return NO_CATALOG_TYPE;
} else {
return catalogType;
diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestCatalogs.java b/mr/src/test/java/org/apache/iceberg/mr/TestCatalogs.java
index 0d5b1d6..0e76ea1 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/TestCatalogs.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/TestCatalogs.java
@@ -197,6 +197,7 @@ public class TestCatalogs {
Optional<Catalog> defaultCatalog = Catalogs.loadCatalog(conf, null);
Assert.assertTrue(defaultCatalog.isPresent());
Assertions.assertThat(defaultCatalog.get()).isInstanceOf(HiveCatalog.class);
+ Assert.assertTrue(Catalogs.hiveCatalog(conf, new Properties()));
}
@Test
@@ -205,6 +206,7 @@ public class TestCatalogs {
Optional<Catalog> hiveCatalog = Catalogs.loadCatalog(conf, null);
Assert.assertTrue(hiveCatalog.isPresent());
Assertions.assertThat(hiveCatalog.get()).isInstanceOf(HiveCatalog.class);
+ Assert.assertTrue(Catalogs.hiveCatalog(conf, new Properties()));
}
@Test
@@ -214,6 +216,7 @@ public class TestCatalogs {
Optional<Catalog> hadoopCatalog = Catalogs.loadCatalog(conf, null);
Assert.assertTrue(hadoopCatalog.isPresent());
Assertions.assertThat(hadoopCatalog.get()).isInstanceOf(HadoopCatalog.class);
+ Assert.assertFalse(Catalogs.hiveCatalog(conf, new Properties()));
}
@Test
@@ -223,12 +226,14 @@ public class TestCatalogs {
Optional<Catalog> customHadoopCatalog = Catalogs.loadCatalog(conf, null);
Assert.assertTrue(customHadoopCatalog.isPresent());
Assertions.assertThat(customHadoopCatalog.get()).isInstanceOf(CustomHadoopCatalog.class);
+ Assert.assertFalse(Catalogs.hiveCatalog(conf, new Properties()));
}
@Test
public void testLegacyLoadCatalogLocation() {
conf.set(InputFormatConfig.CATALOG, Catalogs.LOCATION);
Assert.assertFalse(Catalogs.loadCatalog(conf, null).isPresent());
+ Assert.assertFalse(Catalogs.hiveCatalog(conf, new Properties()));
}
@Test
@@ -241,9 +246,13 @@ public class TestCatalogs {
@Test
public void testLoadCatalogDefault() {
- Optional<Catalog> defaultCatalog = Catalogs.loadCatalog(conf, "barCatalog");
+ String catalogName = "barCatalog";
+ Optional<Catalog> defaultCatalog = Catalogs.loadCatalog(conf, catalogName);
Assert.assertTrue(defaultCatalog.isPresent());
Assertions.assertThat(defaultCatalog.get()).isInstanceOf(HiveCatalog.class);
+ Properties properties = new Properties();
+ properties.put(InputFormatConfig.CATALOG_NAME, catalogName);
+ Assert.assertTrue(Catalogs.hiveCatalog(conf, properties));
}
@Test
@@ -254,6 +263,9 @@ public class TestCatalogs {
Optional<Catalog> hiveCatalog = Catalogs.loadCatalog(conf, catalogName);
Assert.assertTrue(hiveCatalog.isPresent());
Assertions.assertThat(hiveCatalog.get()).isInstanceOf(HiveCatalog.class);
+ Properties properties = new Properties();
+ properties.put(InputFormatConfig.CATALOG_NAME, catalogName);
+ Assert.assertTrue(Catalogs.hiveCatalog(conf, properties));
}
@Test
@@ -267,6 +279,9 @@ public class TestCatalogs {
Assert.assertTrue(hadoopCatalog.isPresent());
Assertions.assertThat(hadoopCatalog.get()).isInstanceOf(HadoopCatalog.class);
Assert.assertEquals("HadoopCatalog{name=barCatalog, location=/tmp/mylocation}", hadoopCatalog.get().toString());
+ Properties properties = new Properties();
+ properties.put(InputFormatConfig.CATALOG_NAME, catalogName);
+ Assert.assertFalse(Catalogs.hiveCatalog(conf, properties));
}
@Test
@@ -279,6 +294,9 @@ public class TestCatalogs {
Assert.assertTrue(hadoopCatalog.isPresent());
Assertions.assertThat(hadoopCatalog.get()).isInstanceOf(HadoopCatalog.class);
Assert.assertEquals("HadoopCatalog{name=barCatalog, location=/tmp/mylocation}", hadoopCatalog.get().toString());
+ Properties properties = new Properties();
+ properties.put(InputFormatConfig.CATALOG_NAME, catalogName);
+ Assert.assertFalse(Catalogs.hiveCatalog(conf, properties));
}
@Test
@@ -291,6 +309,9 @@ public class TestCatalogs {
Optional<Catalog> customHadoopCatalog = Catalogs.loadCatalog(conf, catalogName);
Assert.assertTrue(customHadoopCatalog.isPresent());
Assertions.assertThat(customHadoopCatalog.get()).isInstanceOf(CustomHadoopCatalog.class);
+ Properties properties = new Properties();
+ properties.put(InputFormatConfig.CATALOG_NAME, catalogName);
+ Assert.assertFalse(Catalogs.hiveCatalog(conf, properties));
}
@Test