You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2019/07/08 17:25:24 UTC
[flink] branch master updated: [FLINK-13135][hive] unify configs
for meta-objects in HiveCatalog
This is an automated email from the ASF dual-hosted git repository.
bli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new f8e6c83 [FLINK-13135][hive] unify configs for meta-objects in HiveCatalog
f8e6c83 is described below
commit f8e6c83cffbe494d679cecb8dc371cd634b44d2c
Author: bowen.li <bo...@gmail.com>
AuthorDate: Sun Jul 7 12:07:22 2019 -0700
[FLINK-13135][hive] unify configs for meta-objects in HiveCatalog
This PR unifies configs for meta-objects in HiveCatalog, specifically unifies HiveDatabaseConfig, HiveTableConfig, HivePartitionConfig into HiveCatalogConfig.
This closes #9012.
---
.../flink/table/catalog/hive/HiveCatalog.java | 8 +++----
.../table/catalog/hive/HiveCatalogConfig.java | 8 +++++++
.../table/catalog/hive/HiveDatabaseConfig.java | 26 --------------------
.../table/catalog/hive/HivePartitionConfig.java | 27 ---------------------
.../flink/table/catalog/hive/HiveTableConfig.java | 28 ----------------------
.../table/catalog/hive/util/HiveTableUtil.java | 2 +-
.../connectors/hive/HiveTableOutputFormatTest.java | 4 ++--
.../batch/connectors/hive/HiveTableSinkTest.java | 4 ++--
8 files changed, 17 insertions(+), 90 deletions(-)
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
index fea0de4..3b4218d 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
@@ -193,7 +193,7 @@ public class HiveCatalog extends AbstractCatalog {
Map<String, String> properties = hiveDatabase.getParameters();
- properties.put(HiveDatabaseConfig.DATABASE_LOCATION_URI, hiveDatabase.getLocationUri());
+ properties.put(HiveCatalogConfig.DATABASE_LOCATION_URI, hiveDatabase.getLocationUri());
return new CatalogDatabaseImpl(properties, hiveDatabase.getDescription());
}
@@ -221,7 +221,7 @@ public class HiveCatalog extends AbstractCatalog {
Map<String, String> properties = database.getProperties();
- String dbLocationUri = properties.remove(HiveDatabaseConfig.DATABASE_LOCATION_URI);
+ String dbLocationUri = properties.remove(HiveCatalogConfig.DATABASE_LOCATION_URI);
return new Database(
databaseName,
@@ -728,7 +728,7 @@ public class HiveCatalog extends AbstractCatalog {
Map<String, String> properties = hivePartition.getParameters();
- properties.put(HivePartitionConfig.PARTITION_LOCATION, hivePartition.getSd().getLocation());
+ properties.put(HiveCatalogConfig.PARTITION_LOCATION, hivePartition.getSd().getLocation());
String comment = properties.remove(HiveCatalogConfig.COMMENT);
@@ -813,7 +813,7 @@ public class HiveCatalog extends AbstractCatalog {
}
// TODO: handle GenericCatalogPartition
StorageDescriptor sd = hiveTable.getSd().deepCopy();
- sd.setLocation(catalogPartition.getProperties().remove(HivePartitionConfig.PARTITION_LOCATION));
+ sd.setLocation(catalogPartition.getProperties().remove(HiveCatalogConfig.PARTITION_LOCATION));
Map<String, String> properties = new HashMap<>(catalogPartition.getProperties());
properties.put(HiveCatalogConfig.COMMENT, catalogPartition.getComment());
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalogConfig.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalogConfig.java
index 684f50e..2ece551 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalogConfig.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalogConfig.java
@@ -23,5 +23,13 @@ package org.apache.flink.table.catalog.hive;
*/
public class HiveCatalogConfig {
+ // Database related configs
+ public static final String DATABASE_LOCATION_URI = "database.location_uri";
+
+ // Table related configs
public static final String COMMENT = "comment";
+ public static final String DEFAULT_LIST_COLUMN_TYPES_SEPARATOR = ":";
+
+ // Partition related configs
+ public static final String PARTITION_LOCATION = "partition.location";
}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveDatabaseConfig.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveDatabaseConfig.java
deleted file mode 100644
index 87e1714..0000000
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveDatabaseConfig.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.catalog.hive;
-
-/**
- * Configs for database in {@link HiveCatalog}.
- */
-public class HiveDatabaseConfig {
- public static final String DATABASE_LOCATION_URI = "database.location_uri";
-}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HivePartitionConfig.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HivePartitionConfig.java
deleted file mode 100644
index 0551b72..0000000
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HivePartitionConfig.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.catalog.hive;
-
-/**
- * Configs for partition in {@link HiveCatalog}.
- */
-public class HivePartitionConfig {
- public static final String PARTITION_LOCATION = "partition.location";
-
-}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveTableConfig.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveTableConfig.java
deleted file mode 100644
index 2bb833a..0000000
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveTableConfig.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.catalog.hive;
-
-/**
- * Configs for tables in Hive metastore.
- */
-public class HiveTableConfig {
-
- public static final String DEFAULT_LIST_COLUMN_TYPES_SEPARATOR = ":";
-
-}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTableUtil.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTableUtil.java
index c2597a9..58075a9 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTableUtil.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/util/HiveTableUtil.java
@@ -35,7 +35,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import static org.apache.flink.table.catalog.hive.HiveTableConfig.DEFAULT_LIST_COLUMN_TYPES_SEPARATOR;
+import static org.apache.flink.table.catalog.hive.HiveCatalogConfig.DEFAULT_LIST_COLUMN_TYPES_SEPARATOR;
/**
* Utils to for Hive-backed table.
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableOutputFormatTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableOutputFormatTest.java
index 89dfc8a..ead2fda 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableOutputFormatTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableOutputFormatTest.java
@@ -28,7 +28,7 @@ import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.CatalogTableImpl;
import org.apache.flink.table.catalog.ObjectPath;
import org.apache.flink.table.catalog.hive.HiveCatalog;
-import org.apache.flink.table.catalog.hive.HivePartitionConfig;
+import org.apache.flink.table.catalog.hive.HiveCatalogConfig;
import org.apache.flink.table.catalog.hive.HiveTestUtils;
import org.apache.flink.types.Row;
@@ -132,7 +132,7 @@ public class HiveTableOutputFormatTest {
CatalogPartition catalogPartition = hiveCatalog.getPartition(tablePath, new CatalogPartitionSpec(
partSpec.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()))));
- String partitionLocation = catalogPartition.getProperties().get(HivePartitionConfig.PARTITION_LOCATION);
+ String partitionLocation = catalogPartition.getProperties().get(HiveCatalogConfig.PARTITION_LOCATION);
verifyWrittenData(new Path(partitionLocation, "0"), toWrite, 1);
hiveCatalog.dropTable(tablePath, false);
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableSinkTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableSinkTest.java
index 55f5336..e6505df 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableSinkTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/batch/connectors/hive/HiveTableSinkTest.java
@@ -29,7 +29,7 @@ import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.CatalogTableImpl;
import org.apache.flink.table.catalog.ObjectPath;
import org.apache.flink.table.catalog.hive.HiveCatalog;
-import org.apache.flink.table.catalog.hive.HivePartitionConfig;
+import org.apache.flink.table.catalog.hive.HiveCatalogConfig;
import org.apache.flink.table.catalog.hive.HiveTestUtils;
import org.apache.flink.table.types.DataType;
import org.apache.flink.types.Row;
@@ -123,7 +123,7 @@ public class HiveTableSinkTest {
assertEquals(toWrite.size(), partitionSpecs.size());
for (int i = 0; i < toWrite.size(); i++) {
CatalogPartition partition = hiveCatalog.getPartition(tablePath, partitionSpecs.get(i));
- String partitionLocation = partition.getProperties().get(HivePartitionConfig.PARTITION_LOCATION);
+ String partitionLocation = partition.getProperties().get(HiveCatalogConfig.PARTITION_LOCATION);
verifyWrittenData(new Path(partitionLocation, "0"), Collections.singletonList(toWrite.get(i)), 1);
}