[GitHub] [hudi] danny0405 commented on a diff in pull request #5528: [HUDI-3336][HUDI-FLINK]Support custom hadoop config for flink

[GitBox <gi...@apache.org>]

danny0405 commented on code in PR #5528:
URL: https://github.com/apache/hudi/pull/5528#discussion_r867610262


##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -746,16 +762,18 @@ public static Configuration flatOptions(Configuration conf) {
           : key;
       propsMap.put(subKey, value);
     });
-    return fromMap(propsMap);
+    return Configuration.fromMap(propsMap);
   }
 
-  private static boolean hasPropertyOptions(Map<String, String> options) {
-    return options.keySet().stream().anyMatch(k -> k.startsWith(PROPERTIES_PREFIX));
+  private static boolean hasPropertyOptions(Map<String, String> options, String prefix) {
+    return options.keySet().stream().anyMatch(k -> k.startsWith(prefix));
   }
 
   /**
    * Creates a new configuration that is initialized with the options of the given map.
+   * @deprecated Use {@link Configuration#fromMap(Map)} instead.
    */
+  @Deprecated
   public static Configuration fromMap(Map<String, String> map) {

Review Comment:
   We better keep this method for compatibility.



##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -708,32 +709,47 @@ private FlinkOptions() {
 
   // Prefix for Hoodie specific properties.
   private static final String PROPERTIES_PREFIX = "properties.";
-
-  /**
-   * Collects the config options that start with 'properties.' into a 'key'='value' list.
-   */
-  public static Map<String, String> getHoodieProperties(Map<String, String> options) {
-    return getHoodiePropertiesWithPrefix(options, PROPERTIES_PREFIX);
-  }
+  private static final String HADOOP_PREFIX = "hadoop.";
+  private static final  String PARQUET_PREFIX = "parquet.";
 
   /**
    * Collects the config options that start with specified prefix {@code prefix} into a 'key'='value' list.
    */
-  public static Map<String, String> getHoodiePropertiesWithPrefix(Map<String, String> options, String prefix) {
+  public static Map<String, String> getPropertiesWithPrefix(Map<String, String> options, String prefix) {
     final Map<String, String> hoodieProperties = new HashMap<>();
-
-    if (hasPropertyOptions(options)) {
+    if (hasPropertyOptions(options, prefix)) {
       options.keySet().stream()
-          .filter(key -> key.startsWith(PROPERTIES_PREFIX))
+          .filter(key -> key.startsWith(prefix))
           .forEach(key -> {
             final String value = options.get(key);
-            final String subKey = key.substring((prefix).length());
+            final String subKey = key.substring(prefix.length());
             hoodieProperties.put(subKey, value);
           });
     }
     return hoodieProperties;
   }
 
+  public static org.apache.hadoop.conf.Configuration getParquetConf(
+          org.apache.flink.configuration.Configuration options,
+          org.apache.hadoop.conf.Configuration hadoopConf) {
+    org.apache.hadoop.conf.Configuration copy = new org.apache.hadoop.conf.Configuration(hadoopConf);
+    Map<String, String> parquetOptions = getPropertiesWithPrefix(options.toMap(), PARQUET_PREFIX);
+    parquetOptions.forEach((k, v) -> copy.set(PARQUET_PREFIX + k, v));
+    return copy;
+  }
+
+  // Keep the redundant to avoid too many modifications.
+  public static org.apache.hadoop.conf.Configuration getHadoopConf(Configuration conf) {
+    if (conf == null) {
+      return FlinkClientUtil.getHadoopConf();

Review Comment:
   In which case the conf can be  null ?



##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java:
##########
@@ -351,6 +355,7 @@ public static HoodieTableMetaClient createMetaClient(String basePath, org.apache
   /**
    * Creates the meta client.
    */
+  @Deprecated
   public static HoodieTableMetaClient createMetaClient(String basePath) {

Review Comment:
   Can we remove the method directly ?



##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java:
##########
@@ -140,9 +140,13 @@ public static DFSPropertiesConfiguration readConfig(org.apache.hadoop.conf.Confi
     return conf;
   }
 
-  // Keep the redundant to avoid too many modifications.
+  /**
+   * Keep the redundant to avoid too many modifications.
+   * @deprecated Use {@link FlinkOptions#getHadoopConf(Configuration)} instead.
+   */
+  @Deprecated

Review Comment:
   Can we remove the method directly ?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org