You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/02/06 06:44:54 UTC
svn commit: r1067595 - in /hbase/branches/0.90: ./
src/main/java/org/apache/hadoop/hbase/master/
src/main/java/org/apache/hadoop/hbase/regionserver/
src/main/javadoc/org/apache/hadoop/hbase/replication/ src/site/xdoc/
Author: stack
Date: Sun Feb 6 05:44:53 2011
New Revision: 1067595
URL: http://svn.apache.org/viewvc?rev=1067595&view=rev
Log:
HBASE-3500 Documentation update for replication; revert.. .I applied too much
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.90/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html
hbase/branches/0.90/src/site/xdoc/replication.xml
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1067595&r1=1067594&r2=1067595&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Sun Feb 6 05:44:53 2011
@@ -20,7 +20,6 @@ Release 0.90.1 - Unreleased
HBASE-3502 Can't open region because can't open .regioninfo because
AlreadyBeingCreatedException
HBASE-3501 Remove the deletion limit in LogCleaner
- HBASE-3500 Documentation update for replication
IMPROVEMENTS
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1067595&r1=1067594&r2=1067595&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Sun Feb 6 05:44:53 2011
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -54,8 +55,8 @@ import org.apache.hadoop.hbase.catalog.M
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.MetaScanner;
-import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
@@ -598,10 +599,23 @@ implements HMasterInterface, HMasterRegi
public MapWritable regionServerStartup(final HServerInfo serverInfo,
final long serverCurrentTime)
throws IOException {
+ // Set the ip into the passed in serverInfo. Its ip is more than likely
+ // not the ip that the master sees here. See at end of this method where
+ // we pass it back to the regionserver by setting "hbase.regionserver.address"
+ // Everafter, the HSI combination 'server name' is what uniquely identifies
+ // the incoming RegionServer.
+ InetSocketAddress address = new InetSocketAddress(
+ HBaseServer.getRemoteIp().getHostName(),
+ serverInfo.getServerAddress().getPort());
+ serverInfo.setServerAddress(new HServerAddress(address));
+
// Register with server manager
this.serverManager.regionServerStartup(serverInfo, serverCurrentTime);
// Send back some config info
- return createConfigurationSubset();
+ MapWritable mw = createConfigurationSubset();
+ mw.put(new Text("hbase.regionserver.address"),
+ serverInfo.getServerAddress());
+ return mw;
}
/**
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1067595&r1=1067594&r2=1067595&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Sun Feb 6 05:44:53 2011
@@ -123,10 +123,11 @@ public class ServerManager {
// is, reject the server and trigger its expiration. The next time it comes
// in, it should have been removed from serverAddressToServerInfo and queued
// for processing by ProcessServerShutdown.
- checkIsDead(serverInfo.getServerName(), "STARTUP");
- checkAlreadySameHostPort(serverInfo);
- checkClockSkew(serverInfo, serverCurrentTime);
- recordNewServer(serverInfo, false, null);
+ HServerInfo info = new HServerInfo(serverInfo);
+ checkIsDead(info.getServerName(), "STARTUP");
+ checkAlreadySameHostPort(info);
+ checkClockSkew(info, serverCurrentTime);
+ recordNewServer(info, false, null);
}
/**
@@ -239,21 +240,24 @@ public class ServerManager {
HMsg [] regionServerReport(final HServerInfo serverInfo,
final HMsg [] msgs, final HRegionInfo[] mostLoadedRegions)
throws IOException {
+ // Be careful. This method does returns in the middle.
+ HServerInfo info = new HServerInfo(serverInfo);
+
// Check if dead. If it is, it'll get a 'You Are Dead!' exception.
- checkIsDead(serverInfo.getServerName(), "REPORT");
+ checkIsDead(info.getServerName(), "REPORT");
// If we don't know this server, tell it shutdown.
- HServerInfo storedInfo = this.onlineServers.get(serverInfo.getServerName());
+ HServerInfo storedInfo = this.onlineServers.get(info.getServerName());
if (storedInfo == null) {
// Maybe we already have this host+port combo and its just different
// start code?
- checkAlreadySameHostPort(serverInfo);
+ checkAlreadySameHostPort(info);
// Just let the server in. Presume master joining a running cluster.
// recordNewServer is what happens at the end of reportServerStartup.
// The only thing we are skipping is passing back to the regionserver
// the HServerInfo to use. Here we presume a master has already done
// that so we'll press on with whatever it gave us for HSI.
- recordNewServer(serverInfo, true, null);
+ recordNewServer(info, true, null);
// If msgs, put off their processing but this is not enough because
// its possible that the next time the server reports in, we'll still
// not be up and serving. For example, if a split, we'll need the
@@ -266,7 +270,7 @@ public class ServerManager {
}
// Check startcodes
- if (raceThatShouldNotHappenAnymore(storedInfo, serverInfo)) {
+ if (raceThatShouldNotHappenAnymore(storedInfo, info)) {
return HMsg.STOP_REGIONSERVER_ARRAY;
}
@@ -295,7 +299,7 @@ public class ServerManager {
reply = HMsg.STOP_REGIONSERVER_ARRAY;
}
}
- return processRegionServerAllsWell(serverInfo, mostLoadedRegions, reply);
+ return processRegionServerAllsWell(info, mostLoadedRegions, reply);
}
private boolean raceThatShouldNotHappenAnymore(final HServerInfo storedInfo,
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1067595&r1=1067594&r2=1067595&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Sun Feb 6 05:44:53 2011
@@ -50,6 +50,7 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -75,6 +76,7 @@ import org.apache.hadoop.hbase.Stoppable
import org.apache.hadoop.hbase.UnknownRowLockException;
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.YouAreDeadException;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.catalog.RootLocationEditor;
@@ -134,7 +136,6 @@ import org.apache.hadoop.net.DNS;
import org.apache.zookeeper.KeeperException;
import com.google.common.base.Function;
-import com.google.common.collect.Lists;
/**
* HRegionServer makes a set of HRegions available to clients. It checks in with
@@ -838,7 +839,16 @@ public class HRegionServer implements HR
protected void handleReportForDutyResponse(final MapWritable c) throws IOException {
try {
for (Map.Entry<Writable, Writable> e : c.entrySet()) {
+
String key = e.getKey().toString();
+ // Use the address the master passed us
+ if (key.equals("hbase.regionserver.address")) {
+ HServerAddress hsa = (HServerAddress) e.getValue();
+ LOG.info("Master passed us address to use. Was="
+ + this.serverInfo.getServerAddress() + ", Now=" + hsa.toString());
+ this.serverInfo.setServerAddress(hsa);
+ continue;
+ }
String value = e.getValue().toString();
if (LOG.isDebugEnabled()) {
LOG.debug("Config from master: " + key + "=" + value);
Modified: hbase/branches/0.90/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html?rev=1067595&r1=1067594&r2=1067595&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html (original)
+++ hbase/branches/0.90/src/main/javadoc/org/apache/hadoop/hbase/replication/package.html Sun Feb 6 05:44:53 2011
@@ -92,7 +92,6 @@ to another.
<name>hbase.replication</name>
<value>true</value>
</property></pre>
- deploy the files, and then restart HBase if it was running.
</li>
<li>Run the following command in the master's shell while it's running
<pre>add_peer</pre>
@@ -101,18 +100,6 @@ to another.
to use a different <b>zookeeper.znode.parent</b> since they can't
write in the same folder.
</li>
- <li>
- Once you have a peer, you need to enable replication on your column families.
- One way to do it is to alter the table and to set the scope like this:
- <pre>
- disable 'your_table'
- alter 'your_table', {NAME => 'family_name', REPLICATION_SCOPE => '1'}
- enable 'your_table'
- </pre>
- Currently, a scope of 0 (default) means that it won't be replicated and a
- scope of 1 means it's going to be. In the future, different scope can be
- used for routing policies.
- </li>
</ol>
You can confirm that your setup works by looking at any region server's log
Modified: hbase/branches/0.90/src/site/xdoc/replication.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/site/xdoc/replication.xml?rev=1067595&r1=1067594&r2=1067595&view=diff
==============================================================================
--- hbase/branches/0.90/src/site/xdoc/replication.xml (original)
+++ hbase/branches/0.90/src/site/xdoc/replication.xml Sun Feb 6 05:44:53 2011
@@ -43,10 +43,7 @@
other well known solutions like MySQL master/slave replication where
thereâs only one bin log to keep track of. One master cluster can
replicate to any number of slave clusters, and each region server will
- participate to replicate their own stream of edits. For more information
- on the different properties of master/slave replication and other types
- of replication, please consult <a href="http://highscalability.com/blog/2009/8/24/how-google-serves-data-from-multiple-datacenters.html">
- How Google Serves Data From Multiple Datacenters</a>.
+ participate to replicate their own stream of edits.
</p>
<p>
The replication is done asynchronously, meaning that the clusters can
@@ -76,17 +73,6 @@
</p>
<img src="images/replication_overview.png"/>
</section>
- <section name="Enabling replication">
- <p>
- The guide on enabling and using cluster replication is contained
- in the API documentation shipped with your HBase distribution.
- </p>
- <p>
- The most up-to-date documentation is
- <a href="apidocs/org/apache/hadoop/hbase/replication/package-summary.html#requirements">
- available at this address</a>.
- </p>
- </section>
<section name="Life of a log edit">
<p>
The following sections describe the life of a single edit going from a
@@ -364,6 +350,15 @@
</section>
</section>
<section name="FAQ">
+ <section name="Why do all clusters need to be in the same timezone?">
+ <p>
+ Suppose an edit to cell X happens in a EST cluster, then 2 minutes
+ later a new edits happens to the same cell in a PST cluster and that
+ both clusters are in a master-master replication. The second edit is
+ considered younger, so the first will always hide it while in fact the
+ second is older.
+ </p>
+ </section>
<section name="GLOBAL means replicate? Any provision to replicate only to cluster X and not to cluster Y? or is that for later?">
<p>
Yes, this is for much later.