[GitHub] [hbase] wchevreuil commented on a diff in pull request #4673: HBASE-27265 : Tool to read StoreFileTrackerFile

[GitBox <gi...@apache.org>]

wchevreuil commented on code in PR #4673:
URL: https://github.com/apache/hbase/pull/4673#discussion_r935341877


##########
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFilePrettyPrinter.java:
##########
@@ -0,0 +1,192 @@
+package org.apache.hadoop.hbase.regionserver.storefiletracker;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreContext;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.*;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.zip.CRC32;
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
+@InterfaceStability.Evolving
+public class StoreFileListFilePrettyPrinter extends Configured implements Tool {
+  private static final Logger LOG = LoggerFactory.getLogger(StoreFileListFilePrettyPrinter.class);
+
+  private Options options = new Options();
+
+  private final String fileOption = "f";
+  private final String columnFamilyOption = "cf";
+  private final String regionOption = "r";
+  private final String tableNameOption = "t";
+
+  private String namespace;
+  private String regionName;
+  private String columnFamily;
+  private String tableName;
+  private Path path;
+  private PrintStream err = System.err;
+  private PrintStream out = System.out;
+
+  public StoreFileListFilePrettyPrinter() {
+    super();
+    init();
+  }
+
+  public StoreFileListFilePrettyPrinter(Configuration conf) {
+    super(conf);
+    init();
+  }
+
+  private void init() {
+    OptionGroup files = new OptionGroup();
+    options.addOption(new Option(tableNameOption, "table", true,
+      "Table to scan. Pass table name; e.g. test_table"));
+    options.addOption(new Option(columnFamilyOption, "columnfamily", true,
+      "column family to scan. Pass column family name; e.g. f"));
+    files.addOption(new Option(regionOption, "region", true, "Region to scan. Pass region name; e.g. '3d58e9067bf23e378e68c071f3dd39eb'"));
+    files.addOption(new Option(fileOption, "file", true, "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/hbase:meta/12/34"));
+    options.addOptionGroup(files);
+  }
+
+  public boolean parseOptions(String args[]) throws ParseException, IOException {
+    HelpFormatter formatter = new HelpFormatter();
+    if (args.length == 0) {
+      formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+      return false;
+    }
+
+    CommandLineParser parser = new PosixParser();
+    CommandLine cmd = parser.parse(options, args);
+
+    if (cmd.hasOption(fileOption)) {
+      path = new Path(cmd.getOptionValue(fileOption));
+    } else {
+      regionName = cmd.getOptionValue(regionOption);
+      if(StringUtils.isEmpty(regionName)) {
+        err.println("Region name is not specified.");
+        formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+        System.exit(-1);
+      }
+      columnFamily = cmd.getOptionValue(columnFamilyOption);
+      if(StringUtils.isEmpty(columnFamily)) {
+        err.println("Column family is not specified.");
+        formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+        System.exit(-1);
+      }
+      String tableNameWtihNS = cmd.getOptionValue(tableNameOption);
+      if(StringUtils.isEmpty(tableNameWtihNS)) {
+        err.println("Table name is not specified.");
+        formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+        System.exit(-1);
+      }
+      TableName tn = TableName.valueOf(tableNameWtihNS);
+      namespace = tn.getNamespaceAsString();
+      tableName = tn.getNameAsString();
+    }
+    return true;
+  }
+
+  public int run(String[] args) {
+    if(getConf() == null) {
+      throw new RuntimeException("A Configuration instance must be provided.");
+    }
+    try {
+      CommonFSUtils.setFsDefault(getConf(), CommonFSUtils.getRootDir(getConf()));
+      if (!parseOptions(args)) {
+        return 1;
+      }
+    } catch (IOException ex) {
+      LOG.error("Error parsing command-line options", ex);
+      return 1;
+    } catch (ParseException ex) {
+      LOG.error("Error parsing command-line options", ex);
+      return 1;
+    }
+    FileSystem fs = null;
+    if(path != null) {
+      try {
+        fs = path.getFileSystem(getConf());
+        if(fs.isDirectory(path)) {
+          err.println("ERROR, wrong path given: " + path);
+          return -2;
+        }
+        return print(fs, path);
+      } catch (IOException e) {
+        LOG.error("Error reading " + path, e);
+        return -2;
+      }
+    } else {
+      try {
+        Path root = CommonFSUtils.getRootDir(getConf());
+        Path baseDir = new Path(root, HConstants.BASE_NAMESPACE_DIR);
+        Path nameSpacePath = new Path(baseDir, namespace);
+        Path tablePath = new Path(nameSpacePath, tableName);
+        Path regionPath = new Path(tablePath, regionName);
+        Path cfPath = new Path(regionPath, columnFamily);
+        Path sftPath = new Path(cfPath, StoreFileListFile.TRACK_FILE_DIR);
+
+        fs = FileSystem.newInstance(regionPath.toUri(), getConf());
+
+        RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(sftPath, false);
+
+        while (iterator.hasNext()) {
+          LocatedFileStatus lfs = iterator.next();
+          if(lfs.isFile() && StoreFileListFile.TRACK_FILE_PATTERN.matcher(lfs.getPath().getName()).matches()) {
+            out.println("Printing contents for file " + lfs.getPath().toString());
+            int ret = print(fs, lfs.getPath());
+            if(ret < 0) {
+              return ret;
+            }

Review Comment:
    nit: maybe worth still try to print the remaining files before returning error code?



##########
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFilePrettyPrinter.java:
##########
@@ -0,0 +1,192 @@
+package org.apache.hadoop.hbase.regionserver.storefiletracker;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreContext;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.*;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.zip.CRC32;
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
+@InterfaceStability.Evolving
+public class StoreFileListFilePrettyPrinter extends Configured implements Tool {
+  private static final Logger LOG = LoggerFactory.getLogger(StoreFileListFilePrettyPrinter.class);
+
+  private Options options = new Options();
+
+  private final String fileOption = "f";
+  private final String columnFamilyOption = "cf";
+  private final String regionOption = "r";
+  private final String tableNameOption = "t";
+
+  private String namespace;
+  private String regionName;
+  private String columnFamily;
+  private String tableName;
+  private Path path;
+  private PrintStream err = System.err;
+  private PrintStream out = System.out;
+
+  public StoreFileListFilePrettyPrinter() {
+    super();
+    init();
+  }
+
+  public StoreFileListFilePrettyPrinter(Configuration conf) {
+    super(conf);
+    init();
+  }
+
+  private void init() {
+    OptionGroup files = new OptionGroup();
+    options.addOption(new Option(tableNameOption, "table", true,
+      "Table to scan. Pass table name; e.g. test_table"));
+    options.addOption(new Option(columnFamilyOption, "columnfamily", true,
+      "column family to scan. Pass column family name; e.g. f"));
+    files.addOption(new Option(regionOption, "region", true, "Region to scan. Pass region name; e.g. '3d58e9067bf23e378e68c071f3dd39eb'"));
+    files.addOption(new Option(fileOption, "file", true, "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/hbase:meta/12/34"));
+    options.addOptionGroup(files);
+  }
+
+  public boolean parseOptions(String args[]) throws ParseException, IOException {
+    HelpFormatter formatter = new HelpFormatter();
+    if (args.length == 0) {
+      formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+      return false;
+    }
+
+    CommandLineParser parser = new PosixParser();
+    CommandLine cmd = parser.parse(options, args);
+
+    if (cmd.hasOption(fileOption)) {
+      path = new Path(cmd.getOptionValue(fileOption));
+    } else {
+      regionName = cmd.getOptionValue(regionOption);
+      if(StringUtils.isEmpty(regionName)) {
+        err.println("Region name is not specified.");
+        formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+        System.exit(-1);
+      }
+      columnFamily = cmd.getOptionValue(columnFamilyOption);
+      if(StringUtils.isEmpty(columnFamily)) {
+        err.println("Column family is not specified.");
+        formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+        System.exit(-1);
+      }
+      String tableNameWtihNS = cmd.getOptionValue(tableNameOption);
+      if(StringUtils.isEmpty(tableNameWtihNS)) {
+        err.println("Table name is not specified.");
+        formatter.printHelp("sft [--file=</path/to/tracker/file> | --table=<namespace:tablename|tablename> --region=<regionname> [--columnFamily=<columnfamily>] ]", options, true);
+        System.exit(-1);
+      }
+      TableName tn = TableName.valueOf(tableNameWtihNS);
+      namespace = tn.getNamespaceAsString();
+      tableName = tn.getNameAsString();
+    }
+    return true;
+  }
+
+  public int run(String[] args) {
+    if(getConf() == null) {
+      throw new RuntimeException("A Configuration instance must be provided.");
+    }
+    try {
+      CommonFSUtils.setFsDefault(getConf(), CommonFSUtils.getRootDir(getConf()));
+      if (!parseOptions(args)) {
+        return 1;
+      }
+    } catch (IOException ex) {
+      LOG.error("Error parsing command-line options", ex);
+      return 1;
+    } catch (ParseException ex) {
+      LOG.error("Error parsing command-line options", ex);
+      return 1;
+    }
+    FileSystem fs = null;
+    if(path != null) {
+      try {
+        fs = path.getFileSystem(getConf());
+        if(fs.isDirectory(path)) {
+          err.println("ERROR, wrong path given: " + path);
+          return -2;

Review Comment:
   nit: follow posix standards for return errors (>0)?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@hbase.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org