You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 19:44:24 UTC
svn commit: r1181960 - in
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase:
client/HBaseLocalityCheck.java master/HMaster.java
master/RegionManager.java util/FSUtils.java
Author: nspiegelberg
Date: Tue Oct 11 17:44:24 2011
New Revision: 1181960
URL: http://svn.apache.org/viewvc?rev=1181960&view=rev
Log:
Allow the regionsLocality maping to be written/loaded from the disk, instead of recalculating it each time.
Test Plan:
(i) mvn package
(ii) deploy and run on the dev cluster
(a) ensure that the file was created when run for the first time (when it didn't exist).
(b) stop+start HBase. ensure that the snapshot is loaded from the file,
and not recomputed. Also check that the modified time is not changed.
(c) set the timeout value to something small and check that the snapshot
is recreated.
(d) delete the file and check that the file is recreated
Reviewers: kannan, kranganathan, liyintang, bogdan
Reviewed By: kannan
CC: nspiegelberg, , aaiyer, kranganathan, kannan, bogdan
Differential Revision: 306796
Conflicts:
src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseLocalityCheck.java (contents, props changed)
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (contents, props changed)
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (contents, props changed)
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (contents, props changed)
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseLocalityCheck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseLocalityCheck.java?rev=1181960&r1=1181959&r2=1181960&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseLocalityCheck.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseLocalityCheck.java Tue Oct 11 17:44:24 2011
@@ -1,5 +1,7 @@
package org.apache.hadoop.hbase.client;
+import java.io.DataOutputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
@@ -14,21 +16,20 @@ import org.apache.commons.cli.ParseExcep
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseFsck.HbckInfo;
-import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
public class HBaseLocalityCheck {
private static final Log LOG = LogFactory.getLog(HBaseLocalityCheck.class
.getName());
- private final FileSystem fs;
- private final Path rootdir;
- private Map<String, String> preferredRegionToRegionServerMapping = null;
+ private Map<Writable, Writable> preferredRegionToRegionServerMapping = null;
private Configuration conf;
/**
* The table we want to get locality for, or null in case we wanted a check
@@ -48,8 +49,6 @@ public class HBaseLocalityCheck {
*/
public HBaseLocalityCheck(Configuration conf, final String tableName) throws IOException {
this.conf = conf;
- this.rootdir = FSUtils.getRootDir(conf);
- this.fs = FileSystem.get(conf);
this.tableName = tableName;
}
@@ -74,26 +73,25 @@ public class HBaseLocalityCheck {
LOG.info("Locality information by region");
// Get the locality info for each region by scanning the file system
- preferredRegionToRegionServerMapping = FSUtils
- .getRegionLocalityMappingFromFS(fs, rootdir,
- conf.getInt("hbase.client.localityCheck.threadPoolSize", 2), conf,
- tableName);
+ preferredRegionToRegionServerMapping = HMaster.reevaluateRegionLocality(conf,
+ tableName,
+ conf.getInt("hbase.client.localityCheck.threadPoolSize", 2));
Map<String, AtomicInteger> tableToRegionCountMap =
new HashMap<String, AtomicInteger>();
Map<String, AtomicInteger> tableToRegionsWithLocalityMap =
new HashMap<String, AtomicInteger>();
- for (Map.Entry<String, String> entry :
+ for (Map.Entry<Writable, Writable> entry :
preferredRegionToRegionServerMapping.entrySet()) {
// get region name and table
- String name = entry.getKey();
+ String name = ((Text)entry.getKey()).toString();
int spliterIndex =name.lastIndexOf(":");
String regionName = name.substring(spliterIndex+1);
String tableName = name.substring(0, spliterIndex);
//get region server hostname
- String bestHostName = entry.getValue();
+ String bestHostName = ((Text)entry.getValue()).toString();
localityMatch = false;
HbckInfo region = regionInfo.get(regionName);
if (region != null && region.deployedOn != null &&
@@ -135,8 +133,10 @@ public class HBaseLocalityCheck {
" ;" + " # Local Regions " + totalRegionsWithLocality + " rate = "
+ rate + " %");
}
+
}
+
public static void main(String[] args) throws IOException,
InterruptedException {
long startTime = System.currentTimeMillis();
Propchange: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HBaseLocalityCheck.java
------------------------------------------------------------------------------
svn:executable = *
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1181960&r1=1181959&r2=1181960&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Oct 11 17:44:24 2011
@@ -19,7 +19,14 @@
*/
package org.apache.hadoop.hbase.master;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
@@ -125,6 +132,7 @@ public class HMaster extends Thread impl
//instance into web context.
public static final String MASTER = "master";
private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
+ private static final String LOCALITY_SNAPSHOT_FILE_NAME = "regionLocality-snapshot";
// We start out with closed flag on. Its set to off after construction.
// Use AtomicBoolean rather than plain boolean because we want other threads
@@ -173,7 +181,7 @@ public class HMaster extends Thread impl
boolean isClusterStartup;
private long masterStartupTime = 0;
- private Map<String, String> preferredRegionToRegionServerMapping = null;
+ private MapWritable preferredRegionToRegionServerMapping = null;
private long applyPreferredAssignmentPeriod = 0l;
private long holdRegionForBestLocalityPeriod = 0l;
@@ -297,7 +305,7 @@ public class HMaster extends Thread impl
return this.masterStartupTime;
}
- public Map<String, String> getPreferredRegionToRegionServerMapping() {
+ public MapWritable getPreferredRegionToRegionServerMapping() {
return preferredRegionToRegionServerMapping;
}
@@ -634,23 +642,115 @@ public class HMaster extends Thread impl
this.holdRegionForBestLocalityPeriod =
conf.getLong("hbase.master.holdRegionForBestLocality.period",
1 * 60 * 1000);
- LOG.debug("get preferredRegionToHostMapping; expecting pause here");
- try {
- this.preferredRegionToRegionServerMapping = FSUtils
- .getRegionLocalityMappingFromFS(fs, rootdir,
- conf.getInt("hbase.master.localityCheck.threadPoolSize", 5),
- conf);
- } catch (Exception e) {
- LOG.error("Got unexpected exception when getting " +
- "preferredRegionToHostMapping : " + e.toString());
- // do not pause the master's construction
- preferredRegionToRegionServerMapping = null;
+
+ // try to get the locality map from disk
+ this.preferredRegionToRegionServerMapping = getRegionLocalityFromSnapshot(conf);
+
+ // if we were not successful, let's reevaluate it
+ if (this.preferredRegionToRegionServerMapping == null) {
+ this.preferredRegionToRegionServerMapping = reevaluateRegionLocality(conf, null, conf.getInt("hbase.master.localityCheck.threadPoolSize", 5));
}
+
}
// get the start time stamp after scanning the dfs
masterStartupTime = System.currentTimeMillis();
}
+ public static MapWritable getRegionLocalityFromSnapshot(Configuration conf) {
+ String region_assignment_snapshot_dir =
+ conf.get("hbase.tmp.dir");
+ if (region_assignment_snapshot_dir == null) {
+ return null;
+ }
+
+ String region_assignment_snapshot =
+ region_assignment_snapshot_dir + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
+
+ long refresh_interval =
+ conf.getLong("hbase.master.regionLocality.snapshot.validity_time_ms",
+ 24 * 60 * 60 * 1000);
+
+ File snapshotFile = new File(region_assignment_snapshot);
+ try {
+ if (!snapshotFile.exists()) {
+ LOG.info("preferredRegionToRegionServerMapping snapshot not found. File Path: "
+ + region_assignment_snapshot);
+ return null;
+ }
+
+ long time_elapsed = System.currentTimeMillis() - snapshotFile.lastModified();
+
+ MapWritable regionLocalityMap = null;
+ if (time_elapsed < refresh_interval) {
+ // load the information from disk
+ LOG.debug("Loading preferredRegionToRegionServerMapping from "
+ + region_assignment_snapshot);
+ regionLocalityMap = new MapWritable();
+ regionLocalityMap.readFields(
+ new DataInputStream(new FileInputStream(region_assignment_snapshot)));
+ return regionLocalityMap;
+ }
+ else {
+ LOG.info("Too long since last evaluated region-assignments. "
+ + "Ignoring saved region-assignemnt."
+ + " time_elapsed (ms) = " + time_elapsed + " refresh_interval is " + refresh_interval);
+ }
+ return null;
+ }
+ catch (IOException e) {
+ LOG.error("Error loading the preferredRegionToRegionServerMapping file: " +
+ region_assignment_snapshot + " from Disk : " + e.toString());
+ // do not pause the master's construction
+ return null;
+ }
+
+ }
+
+ /*
+ * Save a copy of the MapWritable regionLocalityMap.
+ * The exact location to be stored is fetched from the Configuration given:
+ * ${hbase.tmp.dir}/regionLocality-snapshot
+ */
+ public static MapWritable reevaluateRegionLocality(Configuration conf, String tablename, int poolSize) {
+ MapWritable regionLocalityMap = null;
+
+ LOG.debug("Evaluate preferredRegionToRegionServerMapping; expecting pause here");
+ try {
+ regionLocalityMap = FSUtils
+ .getRegionLocalityMappingFromFS(FileSystem.get(conf), FSUtils.getRootDir(conf),
+ poolSize,
+ conf,
+ tablename);
+ } catch (Exception e) {
+ LOG.error("Got unexpected exception when evaluating " +
+ "preferredRegionToRegionServerMapping : " + e.toString());
+ // do not pause the master's construction
+ return null;
+ }
+
+ String tmp_path = conf.get("hbase.tmp.dir");
+ if (tmp_path == null) {
+ LOG.info("Could not save preferredRegionToRegionServerMapping " +
+ " config paramater hbase.tmp.dir is not set.");
+ return regionLocalityMap;
+ }
+
+ String region_assignment_snapshot = tmp_path
+ + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
+ // write the preferredRegionAssignment to disk
+ try {
+ LOG.info("Saving preferredRegionToRegionServerMapping " +
+ "to file " + region_assignment_snapshot );
+ regionLocalityMap.write(new DataOutputStream(
+ new FileOutputStream(region_assignment_snapshot)));
+ } catch (IOException e) {
+ LOG.error("Error saving preferredRegionToRegionServerMapping " +
+ "to file " + region_assignment_snapshot + " : " + e.toString());
+ }
+ return regionLocalityMap;
+ }
+
+
/*
* Joins cluster. Checks to see if this instance of HBase is fresh or the
* master was started following a failover. In the second case, it inspects
Propchange: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
------------------------------------------------------------------------------
svn:executable = *
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=1181960&r1=1181959&r2=1181960&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java Tue Oct 11 17:44:24 2011
@@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.util.Pair
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
+import org.apache.hadoop.io.Text;
/**
* Class to manage assigning regions to servers, state of root and meta, etc.
@@ -601,7 +602,7 @@ public class RegionManager {
}
if (assignmentByLocality && !i.isRootRegion() && !i.isMetaRegion()) {
String preferredHost =
- this.master.getPreferredRegionToRegionServerMapping().get(name);
+ this.master.getPreferredRegionToRegionServerMapping().get(new Text(name)).toString();
if (preferredHost != null) {
if (hostName.startsWith(preferredHost)) {
Propchange: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/master/RegionManager.java
------------------------------------------------------------------------------
svn:executable = *
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1181960&r1=1181959&r2=1181960&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Tue Oct 11 17:44:24 2011
@@ -55,7 +55,9 @@ import org.apache.hadoop.hdfs.Distribute
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
+import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
/**
* Utility methods for interacting with the underlying file system.
@@ -715,7 +717,7 @@ public class FSUtils {
* @throws IOException
* in case of file system errors or interrupts
*/
- public static Map<String, String> getRegionLocalityMappingFromFS(
+ public static MapWritable getRegionLocalityMappingFromFS(
final FileSystem fs, final Path rootPath, int threadPoolSize,
final Configuration conf) throws IOException {
return getRegionLocalityMappingFromFS(fs, rootPath, threadPoolSize, conf,
@@ -740,13 +742,12 @@ public class FSUtils {
* @throws IOException
* in case of file system errors or interrupts
*/
- public static Map<String, String> getRegionLocalityMappingFromFS(
+ public static MapWritable getRegionLocalityMappingFromFS(
final FileSystem fs, final Path rootPath, int threadPoolSize,
final Configuration conf, final String desiredTable)
throws IOException {
// region name to its best locality region server mapping
- Map<String, String> regionToBestLocalityRSMapping =
- new ConcurrentHashMap<String, String>();
+ MapWritable regionToBestLocalityRSMapping = new MapWritable();
long startTime = System.currentTimeMillis();
Path queryPath;
@@ -919,10 +920,10 @@ class FSRegionScanner implements Runnabl
* The locality mapping returned by the above getRegionLocalityMappingFromFS
* method
*/
- static private Map<String, String> regionToBestLocalityRSMapping;
+ static private MapWritable regionToBestLocalityRSMapping;
static void setRegionToBestLocalityRSMapping(
- Map<String, String> regionToBestLocalityRSMapping) {
+ MapWritable regionToBestLocalityRSMapping) {
FSRegionScanner.regionToBestLocalityRSMapping =
regionToBestLocalityRSMapping;
}
@@ -1019,8 +1020,9 @@ class FSRegionScanner implements Runnabl
hostToRun = hostToRun.substring(0, hostToRun.length()-1);
}
String name = tableName + ":" + regionPath.getName();
- regionToBestLocalityRSMapping.put(name,hostToRun);
-
+ synchronized (regionToBestLocalityRSMapping) {
+ regionToBestLocalityRSMapping.put(new Text(name), new Text(hostToRun));
+ }
this.numerOfFinishedRegions++;
} catch (IOException e) {
LOG.warn("Problem scanning file system", e);
Propchange: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
------------------------------------------------------------------------------
svn:executable = *