svn commit: r1519030 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/master/balancer/ main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/master/

[dd...@apache.org]

Author: ddas
Date: Fri Aug 30 17:30:50 2013
New Revision: 1519030

URL: http://svn.apache.org/r1519030
Log:
HBASE-9389. Fixes the verification part of favored node assignments

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java?rev=1519030&r1=1519029&r2=1519030&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java Fri Aug 30 17:30:50 2013
@@ -582,6 +582,48 @@ public class AssignmentVerificationRepor
     System.out.println("==============================");
   }
 
+  /**
+   * Return the unassigned regions
+   * @return unassigned regions
+   */
+  List<HRegionInfo> getUnassignedRegions() {
+    return unAssignedRegionsList;
+  }
+
+  /**
+   * Return the regions without favored nodes
+   * @return regions without favored nodes
+   */
+  List<HRegionInfo> getRegionsWithoutValidFavoredNodes() {
+    return regionsWithoutValidFavoredNodes;
+  }
+
+  /**
+   * Return the regions not assigned to its favored nodes
+   * @return regions not assigned to its favored nodes
+   */
+  List<HRegionInfo> getNonFavoredAssignedRegions() {
+    return nonFavoredAssignedRegionList;
+  }
+  
+  /**
+   * Return the number of regions assigned to their favored nodes
+   * @return number of regions assigned to their favored nodes
+   */
+  int getTotalFavoredAssignments() {
+    return totalFavoredAssignments;
+  }
+
+  /**
+   * Return the number of regions based on the position (primary/secondary/
+   * tertiary) assigned to their favored nodes
+   * @param position
+   * @return the number of regions
+   */
+  int getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position position) {
+    return favoredNodes[position.ordinal()];
+  }
+
   private void printHServerAddressSet(Set<ServerName> serverSet) {
     if (serverSet == null) {
       return ;

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java?rev=1519030&r1=1519029&r2=1519030&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java Fri Aug 30 17:30:50 2013
@@ -148,10 +148,13 @@ public class RegionPlacementMaintainer {
   }
 
   /**
-   * Verify the region placement is consistent with the assignment plan;
+   * Verify the region placement is consistent with the assignment plan
+   * @param isDetailMode
+   * @return reports
    * @throws IOException
    */
-  public void verifyRegionPlacement(boolean isDetailMode) throws IOException {
+  public List<AssignmentVerificationReport> verifyRegionPlacement(boolean isDetailMode)
+      throws IOException {
     System.out.println("Start to verify the region assignment and " +
         "generate the verification report");
     // Get the region assignment snapshot
@@ -165,6 +168,7 @@ public class RegionPlacementMaintainer {
     if (this.enforceLocality == true) {
       regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf);
     }
+    List<AssignmentVerificationReport> reports = new ArrayList<AssignmentVerificationReport>();
     // Iterate all the tables to fill up the verification report
     for (TableName table : tables) {
       if (!this.targetTableSet.isEmpty() &&
@@ -174,7 +178,9 @@ public class RegionPlacementMaintainer {
       AssignmentVerificationReport report = new AssignmentVerificationReport();
       report.fillUp(table, snapshot, regionLocalityMap);
       report.print(isDetailMode);
+      reports.add(report);
     }
+    return reports;
   }
 
   /**

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java?rev=1519030&r1=1519029&r2=1519030&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java Fri Aug 30 17:30:50 2013
@@ -87,7 +87,7 @@ public class FavoredNodesPlan {
       return null;
     }
     for (Position p : Position.values()) {
-      if (favoredNodes.get(p.ordinal()).equals(server)) {
+      if (ServerName.isSameHostnameAndPort(favoredNodes.get(p.ordinal()),server)) {
         return p;
       }
     }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java?rev=1519030&r1=1519029&r2=1519030&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java Fri Aug 30 17:30:50 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.BlockLocatio
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
 
 /**
  * Thread that walks over the filesystem, and computes the mappings
@@ -89,7 +90,8 @@ class FSRegionScanner implements Runnabl
           // skip because this is not a CF directory
           continue;
         }
-        if (cfStatus.getPath().getName().startsWith(".")) {
+        if (cfStatus.getPath().getName().startsWith(".") ||
+            HConstants.HBASE_NON_USER_TABLE_DIRS.contains(cfStatus.getPath().getName())) {
           continue;
         }
         FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1519030&r1=1519029&r2=1519030&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Fri Aug 30 17:30:50 2013
@@ -1785,9 +1785,9 @@ public abstract class FSUtils {
     Path queryPath;
     // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/*
     if (null == desiredTable) {
-      queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR), "/*/*/");
+      queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/");
     } else {
-      queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)), "/*/");
+      queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/");
     }
 
     // reject all paths that are not appropriate
@@ -1805,11 +1805,6 @@ public abstract class FSUtils {
           return false;
         }
 
-        // not part of a table?
-        if (!parent.getName().equals(TableName.META_TABLE_NAME.getQualifierAsString())) {
-          return false;
-        }
-
         String regionName = path.getName();
         if (null == regionName) {
           return false;

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java?rev=1519030&r1=1519029&r2=1519030&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java Fri Aug 30 17:30:50 2013
@@ -210,7 +210,8 @@ public class TestRegionPlacement {
 
   @Test
   public void testRegionPlacement() throws Exception {
-    byte[] table = Bytes.toBytes("testRegionAssignment");
+    String tableStr = "testRegionAssignment";
+    byte[] table = Bytes.toBytes(tableStr);
     // Create a table with REGION_NUM regions.
     createTable(table, REGION_NUM);
 
@@ -249,8 +250,36 @@ public class TestRegionPlacement {
     rp.updateAssignmentPlan(shuffledPlan);
 
     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
+
+    // also verify that the AssignmentVerificationReport has the correct information
+    RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
+    // we are interested in only one table (and hence one report)
+    rp.setTargetTableName(new String[]{tableStr});
+    List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
+    AssignmentVerificationReport report = reports.get(0);
+    assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
+    assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
+    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
+    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
+    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
+    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
+    assertTrue(report.getUnassignedRegions().size() == 0);
+
     // Check when a RS stops, the regions get assigned to their secondary/tertiary
     killRandomServerAndVerifyAssignment();
+
+    // also verify that the AssignmentVerificationReport has the correct information
+    reports = rp.verifyRegionPlacement(false);
+    report = reports.get(0);
+    assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
+    assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
+    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
+    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0 && 
+        (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
+        || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
+    assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
+        report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
+        report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
   }