[GitHub] [hudi] felixYyu commented on a diff in pull request #5064: [HUDI-3654] Add new module `hudi-metaserver`

[GitBox <gi...@apache.org>]

felixYyu commented on code in PR #5064:
URL: https://github.com/apache/hudi/pull/5064#discussion_r990579376


##########
hudi-common/src/main/java/org/apache/hudi/common/table/catalog/FileBasedMetaClient.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.catalog;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.fs.FailSafeConsistencyGuard;
+import org.apache.hudi.common.fs.FileSystemRetryConfig;
+import org.apache.hudi.common.fs.HoodieRetryWrapperFileSystem;
+import org.apache.hudi.common.fs.HoodieWrapperFileSystem;
+import org.apache.hudi.common.fs.NoOpConsistencyGuard;
+import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.TimelineLayout;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.exception.TableNotFoundException;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class FileBasedMetaClient implements HoodieMetaClient, Serializable {
+  private static final long serialVersionUID = 1L;
+  private static final Logger LOG = LogManager.getLogger(FileBasedMetaClient.class);
+  public static final String METAFOLDER_NAME = ".hoodie";
+  public static final String AUXILIARYFOLDER_NAME = METAFOLDER_NAME + Path.SEPARATOR + ".aux";
+  public static final String SCHEMA_FOLDER_NAME = ".schema";
+
+  private SerializableConfiguration hadoopConf;
+  private ConsistencyGuardConfig consistencyGuardConfig;
+  private FileSystemRetryConfig fileSystemRetryConfig;
+
+  public FileBasedMetaClient(SerializableConfiguration hadoopConf) {
+    this.hadoopConf = hadoopConf;
+    this.consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build();
+    this.fileSystemRetryConfig = FileSystemRetryConfig.newBuilder().build();
+  }
+
+  public FileBasedMetaClient(SerializableConfiguration hadoopConf, ConsistencyGuardConfig consistencyGuardConfig, FileSystemRetryConfig fileSystemRetryConfig) {
+    this.hadoopConf = hadoopConf;
+    this.consistencyGuardConfig = consistencyGuardConfig;
+    this.fileSystemRetryConfig = fileSystemRetryConfig;
+  }
+
+  public HoodieWrapperFileSystem getFs(String basePath) {
+    FileSystem fileSystem = FSUtils.getFs(new Path(basePath, METAFOLDER_NAME), hadoopConf.newCopy());
+
+    if (fileSystemRetryConfig.isFileSystemActionRetryEnable()) {
+      fileSystem = new HoodieRetryWrapperFileSystem(fileSystem,
+          fileSystemRetryConfig.getMaxRetryIntervalMs(),
+          fileSystemRetryConfig.getMaxRetryNumbers(),
+          fileSystemRetryConfig.getInitialRetryIntervalMs(),
+          fileSystemRetryConfig.getRetryExceptions());
+    }
+    ValidationUtils.checkArgument(!(fileSystem instanceof HoodieWrapperFileSystem),
+        "File System not expected to be that of HoodieWrapperFileSystem");
+    return new HoodieWrapperFileSystem(fileSystem,
+        consistencyGuardConfig.isConsistencyCheckEnabled()
+            ? new FailSafeConsistencyGuard(fileSystem, consistencyGuardConfig)
+            : new NoOpConsistencyGuard());
+  }
+
+  public HoodieTableConfig getHoodieTableConfig(String basePath, String payloadClass) {
+    HoodieWrapperFileSystem fs = getFs(basePath);
+    Path metaPath = new Path(basePath, METAFOLDER_NAME);
+    TableNotFoundException.checkTableValidity(fs, new Path(basePath), metaPath);
+    return new HoodieTableConfig(fs, metaPath.toString(), payloadClass);
+  }
+
+  public static HoodieTableConfig getHoodieTableConfig(String basePath, HoodieWrapperFileSystem fs) {
+    Path metaPath = new Path(basePath, METAFOLDER_NAME);
+    TableNotFoundException.checkTableValidity(fs, new Path(basePath), metaPath);
+    return new HoodieTableConfig(fs, metaPath.toString(), null);
+  }
+
+  /**
+   * Helper method to scan all hoodie-instant metafiles and construct HoodieInstant objects.
+   *
+   * @param basePath table base path
+   * @param includedExtensions Included hoodie extensions
+   * @param applyLayoutVersionFilters Depending on Timeline layout version, if there are multiple states for the same
+   * action instant, only include the highest state
+   * @return List of Hoodie Instants generated
+   * @throws IOException in case of failure
+   */
+  public List<HoodieInstant> scanHoodieInstantsFromFileSystem(String basePath, Set<String> includedExtensions,
+      boolean applyLayoutVersionFilters, TimelineLayoutVersion timelineLayoutVersion) throws IOException {
+    Path timelinePath = new Path(basePath, METAFOLDER_NAME);
+    Stream<HoodieInstant> instantStream = Arrays.stream(
+        FileBasedMetaClient
+            .scanFiles(getFs(basePath), timelinePath, path -> {
+              // Include only the meta files with extensions that needs to be included
+              String extension = HoodieInstant.getTimelineFileExtension(path.getName());
+              return includedExtensions.contains(extension);
+            })).map(HoodieInstant::new);
+一套

Review Comment:
   ‘一套’？Please test carefully



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org