You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2013/02/14 13:58:21 UTC
svn commit: r1446147 [12/35] - in /hbase/branches/hbase-7290v2: ./ bin/
conf/ dev-support/ hbase-client/ hbase-common/
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/
hbase-common/src/...
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Feb 14 12:58:12 2013
@@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.HColumnDe
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HealthCheckChore;
import org.apache.hadoop.hbase.MasterAdminProtocol;
import org.apache.hadoop.hbase.MasterMonitorProtocol;
import org.apache.hadoop.hbase.MasterNotRunningException;
@@ -78,11 +79,11 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
-import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseServer;
-import org.apache.hadoop.hbase.ipc.ProtocolSignature;
+import org.apache.hadoop.hbase.ipc.HBaseServerRPC;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hadoop.hbase.ipc.UnknownProtocolException;
import org.apache.hadoop.hbase.master.balancer.BalancerChore;
import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
@@ -96,7 +97,6 @@ import org.apache.hadoop.hbase.master.ha
import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
-import org.apache.hadoop.hbase.master.handler.TableEventHandler;
import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
@@ -337,6 +337,9 @@ Server {
// monitor for snapshot of hbase tables
private SnapshotManager snapshotManager;
+ /** The health check chore. */
+ private HealthCheckChore healthCheckChore;
+
/**
* Initializes the HMaster. The steps are as follows:
* <p>
@@ -368,9 +371,9 @@ Server {
}
int numHandlers = conf.getInt("hbase.master.handler.count",
conf.getInt("hbase.regionserver.handler.count", 25));
- this.rpcServer = HBaseRPC.getServer(MasterMonitorProtocol.class, this,
- new Class<?>[]{MasterMonitorProtocol.class,
- MasterAdminProtocol.class, RegionServerStatusProtocol.class},
+ this.rpcServer = HBaseServerRPC.getServer(MasterMonitorProtocol.class, this,
+ new Class<?>[]{MasterMonitorProtocol.class,
+ MasterAdminProtocol.class, RegionServerStatusProtocol.class},
initialIsa.getHostName(), // BindAddress is IP we got for this server.
initialIsa.getPort(),
numHandlers,
@@ -416,6 +419,13 @@ Server {
this.masterCheckCompression = conf.getBoolean("hbase.master.check.compression", true);
this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
+
+ // Health checker thread.
+ int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
+ HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
+ if (isHealthCheckerConfigured()) {
+ healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
+ }
}
/**
@@ -440,7 +450,8 @@ Server {
while (!amm.isActiveMaster()) {
LOG.debug("Waiting for master address ZNode to be written " +
"(Also watching cluster state node)");
- Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000));
+ Thread.sleep(
+ c.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT));
}
}
@@ -548,8 +559,7 @@ Server {
*/
private void initializeZKBasedSystemTrackers() throws IOException,
InterruptedException, KeeperException {
- this.catalogTracker = createCatalogTracker(this.zooKeeper, this.conf,
- this, conf.getInt("hbase.master.catalog.timeout", 600000));
+ this.catalogTracker = createCatalogTracker(this.zooKeeper, this.conf, this);
this.catalogTracker.start();
this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
@@ -595,9 +605,9 @@ Server {
* @throws IOException
*/
CatalogTracker createCatalogTracker(final ZooKeeperWatcher zk,
- final Configuration conf, Abortable abortable, final int defaultTimeout)
+ final Configuration conf, Abortable abortable)
throws IOException {
- return new CatalogTracker(zk, conf, abortable, defaultTimeout);
+ return new CatalogTracker(zk, conf, abortable);
}
// Check if we should stop every 100ms
@@ -970,38 +980,12 @@ Server {
return;
}
LOG.info("Forcing splitLog and expire of " + sn);
+ fileSystemManager.splitMetaLog(sn);
fileSystemManager.splitLog(sn);
serverManager.expireServer(sn);
}
@Override
- public ProtocolSignature getProtocolSignature(
- String protocol, long version, int clientMethodsHashCode)
- throws IOException {
- if (MasterMonitorProtocol.class.getName().equals(protocol)) {
- return new ProtocolSignature(MasterMonitorProtocol.VERSION, null);
- } else if (MasterAdminProtocol.class.getName().equals(protocol)) {
- return new ProtocolSignature(MasterAdminProtocol.VERSION, null);
- } else if (RegionServerStatusProtocol.class.getName().equals(protocol)) {
- return new ProtocolSignature(RegionServerStatusProtocol.VERSION, null);
- }
- throw new IOException("Unknown protocol: " + protocol);
- }
-
- public long getProtocolVersion(String protocol, long clientVersion) {
- if (MasterMonitorProtocol.class.getName().equals(protocol)) {
- return MasterMonitorProtocol.VERSION;
- } else if (MasterAdminProtocol.class.getName().equals(protocol)) {
- return MasterAdminProtocol.VERSION;
- } else if (RegionServerStatusProtocol.class.getName().equals(protocol)) {
- return RegionServerStatusProtocol.VERSION;
- }
- // unknown protocol
- LOG.warn("Version requested for unimplemented protocol: "+protocol);
- return -1;
- }
-
- @Override
public TableDescriptors getTableDescriptors() {
return this.tableDescriptors;
}
@@ -1089,6 +1073,11 @@ Server {
this.infoServer.start();
}
+ // Start the health checker
+ if (this.healthCheckChore != null) {
+ Threads.setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker");
+ }
+
// Start allowing requests to happen.
this.rpcServer.openServer();
this.rpcServerOpen = true;
@@ -1124,6 +1113,9 @@ Server {
}
}
if (this.executorService != null) this.executorService.shutdown();
+ if (this.healthCheckChore != null) {
+ this.healthCheckChore.interrupt();
+ }
}
private static Thread getAndStartClusterStatusChore(HMaster master) {
@@ -1579,19 +1571,22 @@ Server {
}
@Override
+ public void deleteTable(final byte[] tableName) throws IOException {
+ checkInitialized();
+ if (cpHost != null) {
+ cpHost.preDeleteTable(tableName);
+ }
+ this.executorService.submit(new DeleteTableHandler(tableName, this, this));
+ if (cpHost != null) {
+ cpHost.postDeleteTable(tableName);
+ }
+ }
+
+ @Override
public DeleteTableResponse deleteTable(RpcController controller, DeleteTableRequest request)
throws ServiceException {
- byte [] tableName = request.getTableName().toByteArray();
try {
- checkInitialized();
- if (cpHost != null) {
- cpHost.preDeleteTable(tableName);
- }
- this.executorService.submit(new DeleteTableHandler(tableName, this, this));
-
- if (cpHost != null) {
- cpHost.postDeleteTable(tableName);
- }
+ deleteTable(request.getTableName().toByteArray());
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
@@ -1626,45 +1621,55 @@ Server {
}
}
+ @Override
+ public void addColumn(final byte[] tableName, final HColumnDescriptor column)
+ throws IOException {
+ checkInitialized();
+ if (cpHost != null) {
+ if (cpHost.preAddColumn(tableName, column)) {
+ return;
+ }
+ }
+ new TableAddFamilyHandler(tableName, column, this, this).process();
+ if (cpHost != null) {
+ cpHost.postAddColumn(tableName, column);
+ }
+ }
+
+ @Override
public AddColumnResponse addColumn(RpcController controller, AddColumnRequest req)
throws ServiceException {
- byte [] tableName = req.getTableName().toByteArray();
- HColumnDescriptor column = HColumnDescriptor.convert(req.getColumnFamilies());
-
try {
- checkInitialized();
- if (cpHost != null) {
- if (cpHost.preAddColumn(tableName, column)) {
- return AddColumnResponse.newBuilder().build();
- }
- }
- new TableAddFamilyHandler(tableName, column, this, this).process();
- if (cpHost != null) {
- cpHost.postAddColumn(tableName, column);
- }
+ addColumn(req.getTableName().toByteArray(),
+ HColumnDescriptor.convert(req.getColumnFamilies()));
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
return AddColumnResponse.newBuilder().build();
}
+ @Override
+ public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor)
+ throws IOException {
+ checkInitialized();
+ checkCompression(descriptor);
+ if (cpHost != null) {
+ if (cpHost.preModifyColumn(tableName, descriptor)) {
+ return;
+ }
+ }
+ new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
+ if (cpHost != null) {
+ cpHost.postModifyColumn(tableName, descriptor);
+ }
+ }
+
+ @Override
public ModifyColumnResponse modifyColumn(RpcController controller, ModifyColumnRequest req)
throws ServiceException {
- byte [] tableName = req.getTableName().toByteArray();
- HColumnDescriptor descriptor = HColumnDescriptor.convert(req.getColumnFamilies());
-
try {
- checkInitialized();
- checkCompression(descriptor);
- if (cpHost != null) {
- if (cpHost.preModifyColumn(tableName, descriptor)) {
- return ModifyColumnResponse.newBuilder().build();
- }
- }
- new TableModifyFamilyHandler(tableName, descriptor, this, this).process();
- if (cpHost != null) {
- cpHost.postModifyColumn(tableName, descriptor);
- }
+ modifyColumn(req.getTableName().toByteArray(),
+ HColumnDescriptor.convert(req.getColumnFamilies()));
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
@@ -1672,21 +1677,25 @@ Server {
}
@Override
+ public void deleteColumn(final byte[] tableName, final byte[] columnName)
+ throws IOException {
+ checkInitialized();
+ if (cpHost != null) {
+ if (cpHost.preDeleteColumn(tableName, columnName)) {
+ return;
+ }
+ }
+ new TableDeleteFamilyHandler(tableName, columnName, this, this).process();
+ if (cpHost != null) {
+ cpHost.postDeleteColumn(tableName, columnName);
+ }
+ }
+
+ @Override
public DeleteColumnResponse deleteColumn(RpcController controller, DeleteColumnRequest req)
throws ServiceException {
- final byte [] tableName = req.getTableName().toByteArray();
- final byte [] columnName = req.getColumnName().toByteArray();
try {
- checkInitialized();
- if (cpHost != null) {
- if (cpHost.preDeleteColumn(tableName, columnName)) {
- return DeleteColumnResponse.newBuilder().build();
- }
- }
- new TableDeleteFamilyHandler(tableName, columnName, this, this).process();
- if (cpHost != null) {
- cpHost.postDeleteColumn(tableName, columnName);
- }
+ deleteColumn(req.getTableName().toByteArray(), req.getColumnName().toByteArray());
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
@@ -1694,20 +1703,23 @@ Server {
}
@Override
+ public void enableTable(final byte[] tableName) throws IOException {
+ checkInitialized();
+ if (cpHost != null) {
+ cpHost.preEnableTable(tableName);
+ }
+ this.executorService.submit(new EnableTableHandler(this, tableName,
+ catalogTracker, assignmentManager, false));
+ if (cpHost != null) {
+ cpHost.postEnableTable(tableName);
+ }
+ }
+
+ @Override
public EnableTableResponse enableTable(RpcController controller, EnableTableRequest request)
throws ServiceException {
- byte [] tableName = request.getTableName().toByteArray();
try {
- checkInitialized();
- if (cpHost != null) {
- cpHost.preEnableTable(tableName);
- }
- this.executorService.submit(new EnableTableHandler(this, tableName,
- catalogTracker, assignmentManager, false));
-
- if (cpHost != null) {
- cpHost.postEnableTable(tableName);
- }
+ enableTable(request.getTableName().toByteArray());
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
@@ -1715,20 +1727,23 @@ Server {
}
@Override
+ public void disableTable(final byte[] tableName) throws IOException {
+ checkInitialized();
+ if (cpHost != null) {
+ cpHost.preDisableTable(tableName);
+ }
+ this.executorService.submit(new DisableTableHandler(this, tableName,
+ catalogTracker, assignmentManager, false));
+ if (cpHost != null) {
+ cpHost.postDisableTable(tableName);
+ }
+ }
+
+ @Override
public DisableTableResponse disableTable(RpcController controller, DisableTableRequest request)
throws ServiceException {
- byte [] tableName = request.getTableName().toByteArray();
try {
- checkInitialized();
- if (cpHost != null) {
- cpHost.preDisableTable(tableName);
- }
- this.executorService.submit(new DisableTableHandler(this, tableName,
- catalogTracker, assignmentManager, false));
-
- if (cpHost != null) {
- cpHost.postDisableTable(tableName);
- }
+ disableTable(request.getTableName().toByteArray());
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
@@ -1771,25 +1786,28 @@ Server {
}
@Override
+ public void modifyTable(final byte[] tableName, final HTableDescriptor descriptor)
+ throws IOException {
+ checkInitialized();
+ checkCompression(descriptor);
+ if (cpHost != null) {
+ cpHost.preModifyTable(tableName, descriptor);
+ }
+ new ModifyTableHandler(tableName, descriptor, this, this).process();
+
+ if (cpHost != null) {
+ cpHost.postModifyTable(tableName, descriptor);
+ }
+ }
+
+ @Override
public ModifyTableResponse modifyTable(RpcController controller, ModifyTableRequest req)
throws ServiceException {
- final byte [] tableName = req.getTableName().toByteArray();
- HTableDescriptor htd = HTableDescriptor.convert(req.getTableSchema());
try {
- checkInitialized();
- checkCompression(htd);
- if (cpHost != null) {
- cpHost.preModifyTable(tableName, htd);
- }
- TableEventHandler tblHandle = new ModifyTableHandler(tableName, htd, this, this);
- this.executorService.submit(tblHandle);
- tblHandle.waitForPersist();
-
- if (cpHost != null) {
- cpHost.postModifyTable(tableName, htd);
- }
+ modifyTable(req.getTableName().toByteArray(),
+ HTableDescriptor.convert(req.getTableSchema()));
} catch (IOException ioe) {
- throw new ServiceException(ioe);
+ throw new ServiceException(ioe);
}
return ModifyTableResponse.newBuilder().build();
}
@@ -2344,7 +2362,7 @@ Server {
String serviceName = call.getServiceName();
String methodName = call.getMethodName();
if (!coprocessorServiceHandlers.containsKey(serviceName)) {
- throw new HBaseRPC.UnknownProtocolException(null,
+ throw new UnknownProtocolException(null,
"No registered master coprocessor service found for name "+serviceName);
}
@@ -2352,7 +2370,7 @@ Server {
Descriptors.ServiceDescriptor serviceDesc = service.getDescriptorForType();
Descriptors.MethodDescriptor methodDesc = serviceDesc.findMethodByName(methodName);
if (methodDesc == null) {
- throw new HBaseRPC.UnknownProtocolException(service.getClass(),
+ throw new UnknownProtocolException(service.getClass(),
"Unknown method "+methodName+" called on master service "+serviceName);
}
@@ -2361,7 +2379,7 @@ Server {
.mergeFrom(call.getRequest()).build();
final Message.Builder responseBuilder =
service.getResponsePrototype(methodDesc).newBuilderForType();
- service.callMethod(methodDesc, controller, execRequest, new RpcCallback<Message>() {
+ service.callMethod(methodDesc, execController, execRequest, new RpcCallback<Message>() {
@Override
public void run(Message message) {
if (message != null) {
@@ -2582,4 +2600,9 @@ Server {
throw new ServiceException(e);
}
}
+
+ private boolean isHealthCheckerConfigured() {
+ String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
+ return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
+ }
}
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java Thu Feb 14 12:58:12 2013
@@ -64,6 +64,7 @@ public class MasterCoprocessorHost
private MasterServices masterServices;
MasterCoprocessorHost(final MasterServices services, final Configuration conf) {
+ this.conf = conf;
this.masterServices = services;
loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY);
}
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Thu Feb 14 12:58:12 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -86,6 +87,18 @@ public class MasterFileSystem {
final SplitLogManager splitLogManager;
private final MasterServices services;
+ private final static PathFilter META_FILTER = new PathFilter() {
+ public boolean accept(Path p) {
+ return HLogUtil.isMetaFile(p);
+ }
+ };
+
+ private final static PathFilter NON_META_FILTER = new PathFilter() {
+ public boolean accept(Path p) {
+ return !HLogUtil.isMetaFile(p);
+ }
+ };
+
public MasterFileSystem(Server master, MasterServices services,
MetricsMaster metricsMaster, boolean masterRecovery)
throws IOException {
@@ -242,7 +255,8 @@ public class MasterFileSystem {
+ " belongs to an existing region server");
}
}
- splitLog(serverNames);
+ splitLog(serverNames, META_FILTER);
+ splitLog(serverNames, NON_META_FILTER);
retrySplitting = false;
} catch (IOException ioe) {
LOG.warn("Failed splitting of " + serverNames, ioe);
@@ -271,8 +285,30 @@ public class MasterFileSystem {
splitLog(serverNames);
}
- public void splitLog(final List<ServerName> serverNames) throws IOException {
+ /**
+ * Specialized method to handle the splitting for meta HLog
+ * @param serverName
+ * @throws IOException
+ */
+ public void splitMetaLog(final ServerName serverName) throws IOException {
long splitTime = 0, splitLogSize = 0;
+ List<ServerName> serverNames = new ArrayList<ServerName>();
+ serverNames.add(serverName);
+ List<Path> logDirs = getLogDirs(serverNames);
+ if (logDirs.isEmpty()) {
+ LOG.info("No meta logs to split");
+ return;
+ }
+ splitLogManager.handleDeadWorkers(serverNames);
+ splitTime = EnvironmentEdgeManager.currentTimeMillis();
+ splitLogSize = splitLogManager.splitLogDistributed(logDirs, META_FILTER);
+ splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
+ if (this.metricsMaster != null) {
+ this.metricsMaster.addSplit(splitTime, splitLogSize);
+ }
+ }
+
+ private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOException {
List<Path> logDirs = new ArrayList<Path>();
for (ServerName serverName: serverNames) {
Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
@@ -290,6 +326,23 @@ public class MasterFileSystem {
}
logDirs.add(splitDir);
}
+ return logDirs;
+ }
+
+ public void splitLog(final List<ServerName> serverNames) throws IOException {
+ splitLog(serverNames, NON_META_FILTER);
+ }
+
+ /**
+ * This method is the base split method that splits HLog files matching a filter.
+ * Callers should pass the appropriate filter for meta and non-meta HLogs.
+ * @param serverNames
+ * @param filter
+ * @throws IOException
+ */
+ public void splitLog(final List<ServerName> serverNames, PathFilter filter) throws IOException {
+ long splitTime = 0, splitLogSize = 0;
+ List<Path> logDirs = getLogDirs(serverNames);
if (logDirs.isEmpty()) {
LOG.info("No logs to split");
@@ -299,7 +352,7 @@ public class MasterFileSystem {
if (distributedLogSplitting) {
splitLogManager.handleDeadWorkers(serverNames);
splitTime = EnvironmentEdgeManager.currentTimeMillis();
- splitLogSize = splitLogManager.splitLogDistributed(logDirs);
+ splitLogSize = splitLogManager.splitLogDistributed(logDirs,filter);
splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
} else {
for(Path logDir: logDirs){
@@ -411,7 +464,7 @@ public class MasterFileSystem {
// if not the cleaner will take care of them.
for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
- HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
+ HFileArchiver.archiveRegion(c, fs, this.rootdir, tabledir, regiondir);
}
}
if (!fs.delete(tmpdir, true)) {
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java Thu Feb 14 12:58:12 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
import com.google.protobuf.Service;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableDescriptors;
@@ -80,6 +81,63 @@ public interface MasterServices extends
throws IOException;
/**
+ * Delete a table
+ * @param tableName The table name
+ * @throws IOException
+ */
+ public void deleteTable(final byte[] tableName) throws IOException;
+
+ /**
+ * Modify the descriptor of an existing table
+ * @param tableName The table name
+ * @param descriptor The updated table descriptor
+ * @throws IOException
+ */
+ public void modifyTable(final byte[] tableName, final HTableDescriptor descriptor)
+ throws IOException;
+
+ /**
+ * Enable an existing table
+ * @param tableName The table name
+ * @throws IOException
+ */
+ public void enableTable(final byte[] tableName) throws IOException;
+
+ /**
+ * Disable an existing table
+ * @param tableName The table name
+ * @throws IOException
+ */
+ public void disableTable(final byte[] tableName) throws IOException;
+
+ /**
+ * Add a new column to an existing table
+ * @param tableName The table name
+ * @param column The column definition
+ * @throws IOException
+ */
+ public void addColumn(final byte[] tableName, final HColumnDescriptor column)
+ throws IOException;
+
+ /**
+ * Modify the column descriptor of an existing column in an existing table
+ * @param tableName The table name
+ * @param descriptor The updated column definition
+ * @throws IOException
+ */
+ public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor)
+ throws IOException;
+
+ /**
+ * Delete a column from an existing table
+ * @param tableName The table name
+ * @param columnName The column name
+ * @throws IOException
+ */
+ public void deleteColumn(final byte[] tableName, final byte[] columnName)
+ throws IOException;
+
+ /**
* @return Return table descriptors implementation.
*/
public TableDescriptors getTableDescriptors();
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java Thu Feb 14 12:58:12 2013
@@ -117,6 +117,18 @@ public class RegionPlan implements Compa
}
@Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ RegionPlan other = (RegionPlan) obj;
+ return compareTo(other) == 0;
+ }
+
+ @Override
public String toString() {
return "hri=" + this.hri.getRegionNameAsString() + ", src=" +
(this.source == null? "": this.source.toString()) +
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Thu Feb 14 12:58:12 2013
@@ -380,7 +380,7 @@ public class ServerManager {
public double getAverageLoad() {
int totalLoad = 0;
int numServers = 0;
- double averageLoad = 0.0;
+ double averageLoad;
for (ServerLoad sl: this.onlineServers.values()) {
numServers++;
totalLoad += sl.getNumberOfRegions();
@@ -680,7 +680,7 @@ public class ServerManager {
*/
private AdminProtocol getServerConnection(final ServerName sn)
throws IOException {
- AdminProtocol admin = this.serverConnections.get(sn.toString());
+ AdminProtocol admin = this.serverConnections.get(sn);
if (admin == null) {
LOG.debug("New connection to " + sn.toString());
admin = this.connection.getAdmin(sn.getHostname(), sn.getPort());
@@ -886,7 +886,7 @@ public class ServerManager {
* To clear any dead server with same host name and port of any online server
*/
void clearDeadServersWithSameHostNameAndPortOfOnlineServer() {
- ServerName sn = null;
+ ServerName sn;
for (ServerName serverName : getOnlineServersList()) {
while ((sn = ServerName.
findServerWithSameHostnamePort(this.deadservers, serverName)) != null) {
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Thu Feb 14 12:58:12 2013
@@ -26,6 +26,7 @@ import static org.apache.hadoop.hbase.ma
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -40,10 +41,11 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.DeserializationException;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.SplitLogTask;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.master.SplitLogManager.TaskFinisher.Status;
@@ -123,6 +125,8 @@ public class SplitLogManager extends Zoo
private volatile Set<ServerName> deadWorkers = null;
private final Object deadWorkersLock = new Object();
+ private Set<String> failedDeletions = null;
+
/**
* Wrapper around {@link #SplitLogManager(ZooKeeperWatcher zkw, Configuration conf,
* Stoppable stopper, MasterServices master, ServerName serverName, TaskFinisher tf)}
@@ -180,6 +184,8 @@ public class SplitLogManager extends Zoo
this.serverName = serverName;
this.timeoutMonitor =
new TimeoutMonitor(conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000), stopper);
+
+ this.failedDeletions = Collections.synchronizedSet(new HashSet<String>());
}
public void finishInitialization(boolean masterRecovery) {
@@ -194,7 +200,7 @@ public class SplitLogManager extends Zoo
}
}
- private FileStatus[] getFileList(List<Path> logDirs) throws IOException {
+ private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws IOException {
List<FileStatus> fileStatus = new ArrayList<FileStatus>();
for (Path hLogDir : logDirs) {
this.fs = hLogDir.getFileSystem(conf);
@@ -202,8 +208,7 @@ public class SplitLogManager extends Zoo
LOG.warn(hLogDir + " doesn't exist. Nothing to do!");
continue;
}
- // TODO filter filenames?
- FileStatus[] logfiles = FSUtils.listStatus(fs, hLogDir, null);
+ FileStatus[] logfiles = FSUtils.listStatus(fs, hLogDir, filter);
if (logfiles == null || logfiles.length == 0) {
LOG.info(hLogDir + " is empty dir, no logs to split");
} else {
@@ -228,6 +233,7 @@ public class SplitLogManager extends Zoo
logDirs.add(logDir);
return splitLogDistributed(logDirs);
}
+
/**
* The caller will block until all the log files of the given region server
* have been processed - successfully split or an error is encountered - by an
@@ -239,9 +245,25 @@ public class SplitLogManager extends Zoo
* @return cumulative size of the logfiles split
*/
public long splitLogDistributed(final List<Path> logDirs) throws IOException {
+ return splitLogDistributed(logDirs, null);
+ }
+
+ /**
+ * The caller will block until all the META log files of the given region server
+ * have been processed - successfully split or an error is encountered - by an
+ * available worker region server. This method must only be called after the
+ * region servers have been brought online.
+ *
+ * @param logDirs List of log dirs to split
+ * @param filter the Path filter to select specific files for considering
+ * @throws IOException If there was an error while splitting any log file
+ * @return cumulative size of the logfiles split
+ */
+ public long splitLogDistributed(final List<Path> logDirs, PathFilter filter)
+ throws IOException {
MonitoredTask status = TaskMonitor.get().createStatus(
"Doing distributed log split in " + logDirs);
- FileStatus[] logfiles = getFileList(logDirs);
+ FileStatus[] logfiles = getFileList(logDirs, filter);
status.setStatus("Checking directory contents...");
LOG.debug("Scheduling batch of logs to split");
SplitLogCounters.tot_mgr_log_split_batch_start.incrementAndGet();
@@ -418,11 +440,12 @@ public class SplitLogManager extends Zoo
}
}
}
- // delete the task node in zk. Keep trying indefinitely - its an async
+ // delete the task node in zk. It's an async
// call and no one is blocked waiting for this node to be deleted. All
// task names are unique (log.<timestamp>) there is no risk of deleting
// a future task.
- deleteNode(path, Long.MAX_VALUE);
+ // if a deletion fails, TimeoutMonitor will retry the same deletion later
+ deleteNode(path, zkretries);
return;
}
@@ -531,6 +554,21 @@ public class SplitLogManager extends Zoo
}
}
+ /**
+ * Helper function to check whether to abandon retries in ZooKeeper AsyncCallback functions
+ * @param statusCode integer value of a ZooKeeper exception code
+ * @param action description message about the retried action
+ * @return true when need to abandon retries otherwise false
+ */
+ private boolean needAbandonRetries(int statusCode, String action) {
+ if (statusCode == KeeperException.Code.SESSIONEXPIRED.intValue()) {
+ LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries for "
+ + "action=" + action);
+ return true;
+ }
+ return false;
+ }
+
private void heartbeat(String path, int new_version, ServerName workerName) {
Task task = findOrCreateOrphanTask(path);
if (new_version != task.last_version) {
@@ -662,8 +700,7 @@ public class SplitLogManager extends Zoo
}
private void deleteNodeFailure(String path) {
- LOG.fatal("logic failure, failing to delete a node should never happen " +
- "because delete has infinite retries");
+ LOG.info("Failed to delete node " + path + " and will retry soon.");
return;
}
@@ -847,7 +884,7 @@ public class SplitLogManager extends Zoo
volatile long last_update;
volatile int last_version;
volatile ServerName cur_worker_name;
- TaskBatch batch;
+ volatile TaskBatch batch;
volatile TerminationStatus status;
volatile int incarnation;
volatile int unforcedResubmits;
@@ -1005,6 +1042,16 @@ public class SplitLogManager extends Zoo
SplitLogCounters.tot_mgr_resubmit_unassigned.incrementAndGet();
LOG.debug("resubmitting unassigned task(s) after timeout");
}
+
+ // Retry previously failed deletes
+ if (failedDeletions.size() > 0) {
+ List<String> tmpPaths = new ArrayList<String>(failedDeletions);
+ for (String tmpPath : tmpPaths) {
+ // deleteNode is an async call
+ deleteNode(tmpPath, zkretries);
+ }
+ failedDeletions.removeAll(tmpPaths);
+ }
}
}
@@ -1019,6 +1066,10 @@ public class SplitLogManager extends Zoo
public void processResult(int rc, String path, Object ctx, String name) {
SplitLogCounters.tot_mgr_node_create_result.incrementAndGet();
if (rc != 0) {
+ if (needAbandonRetries(rc, "Create znode " + path)) {
+ createNodeFailure(path);
+ return;
+ }
if (rc == KeeperException.Code.NODEEXISTS.intValue()) {
// What if there is a delete pending against this pre-existing
// znode? Then this soon-to-be-deleted task znode must be in TASK_DONE
@@ -1058,8 +1109,7 @@ public class SplitLogManager extends Zoo
Stat stat) {
SplitLogCounters.tot_mgr_get_data_result.incrementAndGet();
if (rc != 0) {
- if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
- LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
+ if (needAbandonRetries(rc, "GetData from znode " + path)) {
return;
}
if (rc == KeeperException.Code.NONODE.intValue()) {
@@ -1113,6 +1163,10 @@ public class SplitLogManager extends Zoo
public void processResult(int rc, String path, Object ctx) {
SplitLogCounters.tot_mgr_node_delete_result.incrementAndGet();
if (rc != 0) {
+ if (needAbandonRetries(rc, "Delete znode " + path)) {
+ failedDeletions.add(path);
+ return;
+ }
if (rc != KeeperException.Code.NONODE.intValue()) {
SplitLogCounters.tot_mgr_node_delete_err.incrementAndGet();
Long retry_count = (Long) ctx;
@@ -1120,13 +1174,14 @@ public class SplitLogManager extends Zoo
path + " remaining retries=" + retry_count);
if (retry_count == 0) {
LOG.warn("delete failed " + path);
+ failedDeletions.add(path);
deleteNodeFailure(path);
} else {
deleteNode(path, retry_count - 1);
}
return;
} else {
- LOG.debug(path +
+ LOG.info(path +
" does not exist. Either was created but deleted behind our" +
" back by another pending delete OR was deleted" +
" in earlier retry rounds. zkretries = " + (Long) ctx);
@@ -1151,8 +1206,7 @@ public class SplitLogManager extends Zoo
@Override
public void processResult(int rc, String path, Object ctx, String name) {
if (rc != 0) {
- if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) {
- LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries.");
+ if (needAbandonRetries(rc, "CreateRescan znode " + path)) {
return;
}
Long retry_count = (Long)ctx;
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java Thu Feb 14 12:58:12 2013
@@ -145,13 +145,23 @@ public abstract class CleanerChore<T ext
* @return <tt>true</tt> if the directory was deleted, <tt>false</tt> otherwise.
* @throws IOException if there is an unexpected filesystem error
*/
- private boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
+ public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
if (LOG.isTraceEnabled()) {
LOG.trace("Checking directory: " + toCheck);
}
FileStatus[] children = FSUtils.listStatus(fs, toCheck);
// if the directory doesn't exist, then we are done
- if (children == null) return true;
+ if (children == null) {
+ try {
+ return fs.delete(toCheck, false);
+ } catch (IOException e) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Couldn't delete directory: " + toCheck, e);
+ }
+ }
+ // couldn't delete w/o exception, so we can't return success.
+ return false;
+ }
boolean canDeleteThis = true;
for (FileStatus child : children) {
@@ -168,9 +178,22 @@ public abstract class CleanerChore<T ext
}
}
- // if all the children have been deleted, then we should try to delete this directory. However,
- // don't do so recursively so we don't delete files that have been added since we checked.
- return canDeleteThis ? fs.delete(toCheck, false) : false;
+ // if the directory has children, we can't delete it, so we are done
+ if (!canDeleteThis) return false;
+
+ // otherwise, all the children (that we know about) have been deleted, so we should try to
+ // delete this directory. However, don't do so recursively so we don't delete files that have
+ // been added since we last checked.
+ try {
+ return fs.delete(toCheck, false);
+ } catch (IOException e) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Couldn't delete directory: " + toCheck, e);
+ }
+ }
+
+ // couldn't delete w/o exception, so we can't return success.
+ return false;
}
/**
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java Thu Feb 14 12:58:12 2013
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.util.Envi
/**
* HFile cleaner that uses the timestamp of the hfile to determine if it should be deleted. By
- * default they are allowed to live for {@value TimeToLiveHFileCleaner#DEFAULT_TTL}
+ * default they are allowed to live for {@value #DEFAULT_TTL}
*/
@InterfaceAudience.Private
public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate {
@@ -38,7 +38,7 @@ public class TimeToLiveHFileCleaner exte
public static final Log LOG = LogFactory.getLog(TimeToLiveHFileCleaner.class.getName());
public static final String TTL_CONF_KEY = "hbase.master.hfilecleaner.ttl";
// default ttl = 5 minutes
- private static final long DEFAULT_TTL = 60000 * 5;
+ public static final long DEFAULT_TTL = 60000 * 5;
// Configured time a hfile can be kept after it was moved to the archive
private long ttl;
private FileSystem fs;
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Thu Feb 14 12:58:12 2013
@@ -97,7 +97,7 @@ public class DeleteTableHandler extends
FileSystem fs = mfs.getFileSystem();
for (HRegionInfo hri: regions) {
LOG.debug("Deleting region " + hri.getRegionNameAsString() + " from FS");
- HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+ HFileArchiver.archiveRegion(masterServices.getConfiguration(), fs, mfs.getRootDir(),
tempTableDir, new Path(tempTableDir, hri.getEncodedName()));
}
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java Thu Feb 14 12:58:12 2013
@@ -18,11 +18,17 @@
*/
package org.apache.hadoop.hbase.master.handler;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.DeadServer;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.zookeeper.KeeperException;
/**
* Shutdown handler for the server hosting <code>-ROOT-</code>,
@@ -32,7 +38,7 @@ import org.apache.hadoop.hbase.master.Ma
public class MetaServerShutdownHandler extends ServerShutdownHandler {
private final boolean carryingRoot;
private final boolean carryingMeta;
-
+ private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class);
public MetaServerShutdownHandler(final Server server,
final MasterServices services,
final DeadServer deadServers, final ServerName serverName,
@@ -44,11 +50,118 @@ public class MetaServerShutdownHandler e
}
@Override
+ public void process() throws IOException {
+ try {
+ LOG.info("Splitting META logs for " + serverName);
+ if (this.shouldSplitHlog) {
+ this.services.getMasterFileSystem().splitMetaLog(serverName);
+ }
+ } catch (IOException ioe) {
+ this.services.getExecutorService().submit(this);
+ this.deadServers.add(serverName);
+ throw new IOException("failed log splitting for " +
+ serverName + ", will retry", ioe);
+ }
+
+ // Assign root and meta if we were carrying them.
+ if (isCarryingRoot()) { // -ROOT-
+ // Check again: region may be assigned to other where because of RIT
+ // timeout
+ if (this.services.getAssignmentManager().isCarryingRoot(serverName)) {
+ LOG.info("Server " + serverName
+ + " was carrying ROOT. Trying to assign.");
+ this.services.getAssignmentManager().regionOffline(
+ HRegionInfo.ROOT_REGIONINFO);
+ verifyAndAssignRootWithRetries();
+ } else {
+ LOG.info("ROOT has been assigned to otherwhere, skip assigning.");
+ }
+ }
+
+ // Carrying meta?
+ if (isCarryingMeta()) {
+ // Check again: region may be assigned to other where because of RIT
+ // timeout
+ if (this.services.getAssignmentManager().isCarryingMeta(serverName)) {
+ LOG.info("Server " + serverName
+ + " was carrying META. Trying to assign.");
+ this.services.getAssignmentManager().regionOffline(
+ HRegionInfo.FIRST_META_REGIONINFO);
+ this.services.getAssignmentManager().assignMeta();
+ } else {
+ LOG.info("META has been assigned to otherwhere, skip assigning.");
+ }
+
+ }
+ super.process();
+ }
+ /**
+ * Before assign the ROOT region, ensure it haven't
+ * been assigned by other place
+ * <p>
+ * Under some scenarios, the ROOT region can be opened twice, so it seemed online
+ * in two regionserver at the same time.
+ * If the ROOT region has been assigned, so the operation can be canceled.
+ * @throws InterruptedException
+ * @throws IOException
+ * @throws KeeperException
+ */
+ private void verifyAndAssignRoot()
+ throws InterruptedException, IOException, KeeperException {
+ long timeout = this.server.getConfiguration().
+ getLong("hbase.catalog.verification.timeout", 1000);
+ if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
+ this.services.getAssignmentManager().assignRoot();
+ } else if (serverName.equals(server.getCatalogTracker().getRootLocation())) {
+ throw new IOException("-ROOT- is onlined on the dead server "
+ + serverName);
+ } else {
+ LOG.info("Skip assigning -ROOT-, because it is online on the "
+ + server.getCatalogTracker().getRootLocation());
+ }
+ }
+
+ /**
+ * Failed many times, shutdown processing
+ * @throws IOException
+ */
+ private void verifyAndAssignRootWithRetries() throws IOException {
+ int iTimes = this.server.getConfiguration().getInt(
+ "hbase.catalog.verification.retries", 10);
+
+ long waitTime = this.server.getConfiguration().getLong(
+ "hbase.catalog.verification.timeout", 1000);
+
+ int iFlag = 0;
+ while (true) {
+ try {
+ verifyAndAssignRoot();
+ break;
+ } catch (KeeperException e) {
+ this.server.abort("In server shutdown processing, assigning root", e);
+ throw new IOException("Aborting", e);
+ } catch (Exception e) {
+ if (iFlag >= iTimes) {
+ this.server.abort("verifyAndAssignRoot failed after" + iTimes
+ + " times retries, aborting", e);
+ throw new IOException("Aborting", e);
+ }
+ try {
+ Thread.sleep(waitTime);
+ } catch (InterruptedException e1) {
+ LOG.warn("Interrupted when is the thread sleep", e1);
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted", e1);
+ }
+ iFlag++;
+ }
+ }
+ }
+
boolean isCarryingRoot() {
return this.carryingRoot;
}
- @Override
boolean isCarryingMeta() {
return this.carryingMeta;
}
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java Thu Feb 14 12:58:12 2013
@@ -55,10 +55,10 @@ import org.apache.zookeeper.KeeperExcept
@InterfaceAudience.Private
public class ServerShutdownHandler extends EventHandler {
private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
- private final ServerName serverName;
- private final MasterServices services;
- private final DeadServer deadServers;
- private final boolean shouldSplitHlog; // whether to split HLog or not
+ protected final ServerName serverName;
+ protected final MasterServices services;
+ protected final DeadServer deadServers;
+ protected final boolean shouldSplitHlog; // whether to split HLog or not
public ServerShutdownHandler(final Server server, final MasterServices services,
final DeadServer deadServers, final ServerName serverName,
@@ -91,63 +91,6 @@ public class ServerShutdownHandler exten
}
/**
- * Before assign the ROOT region, ensure it haven't
- * been assigned by other place
- * <p>
- * Under some scenarios, the ROOT region can be opened twice, so it seemed online
- * in two regionserver at the same time.
- * If the ROOT region has been assigned, so the operation can be canceled.
- * @throws InterruptedException
- * @throws IOException
- * @throws KeeperException
- */
- private void verifyAndAssignRoot()
- throws InterruptedException, IOException, KeeperException {
- long timeout = this.server.getConfiguration().
- getLong("hbase.catalog.verification.timeout", 1000);
- if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
- this.services.getAssignmentManager().assignRoot();
- }
- }
-
- /**
- * Failed many times, shutdown processing
- * @throws IOException
- */
- private void verifyAndAssignRootWithRetries() throws IOException {
- int iTimes = this.server.getConfiguration().getInt(
- "hbase.catalog.verification.retries", 10);
-
- long waitTime = this.server.getConfiguration().getLong(
- "hbase.catalog.verification.timeout", 1000);
-
- int iFlag = 0;
- while (true) {
- try {
- verifyAndAssignRoot();
- break;
- } catch (KeeperException e) {
- this.server.abort("In server shutdown processing, assigning root", e);
- throw new IOException("Aborting", e);
- } catch (Exception e) {
- if (iFlag >= iTimes) {
- this.server.abort("verifyAndAssignRoot failed after" + iTimes
- + " times retries, aborting", e);
- throw new IOException("Aborting", e);
- }
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException e1) {
- LOG.warn("Interrupted when is the thread sleep", e1);
- Thread.currentThread().interrupt();
- throw new IOException("Interrupted", e1);
- }
- iFlag++;
- }
- }
- }
-
- /**
* @return True if the server we are processing was carrying <code>-ROOT-</code>
*/
boolean isCarryingRoot() {
@@ -182,30 +125,13 @@ public class ServerShutdownHandler exten
LOG.info("Skipping log splitting for " + serverName);
}
} catch (IOException ioe) {
- this.services.getExecutorService().submit(this);
+ //typecast to SSH so that we make sure that it is the SSH instance that
+ //gets submitted as opposed to MSSH or some other derived instance of SSH
+ this.services.getExecutorService().submit((ServerShutdownHandler)this);
this.deadServers.add(serverName);
throw new IOException("failed log splitting for " +
serverName + ", will retry", ioe);
}
-
- // Assign root and meta if we were carrying them.
- if (isCarryingRoot()) { // -ROOT-
- LOG.info("Server " + serverName +
- " was carrying ROOT. Trying to assign.");
- this.services.getAssignmentManager().
- regionOffline(HRegionInfo.ROOT_REGIONINFO);
- verifyAndAssignRootWithRetries();
- }
-
- // Carrying meta?
- if (isCarryingMeta()) {
- LOG.info("Server " + serverName +
- " was carrying META. Trying to assign.");
- this.services.getAssignmentManager().
- regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
- this.services.getAssignmentManager().assignMeta();
- }
-
// We don't want worker thread in the MetaServerShutdownHandler
// executor pool to block by waiting availability of -ROOT-
// and .META. server. Otherwise, it could run into the following issue:
@@ -430,7 +356,7 @@ public class ServerShutdownHandler exten
if (daughter == null) return 0;
if (isDaughterMissing(catalogTracker, daughter)) {
LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
- MetaEditor.addDaughter(catalogTracker, daughter, null);
+ MetaEditor.addDaughter(catalogTracker, daughter, null, HConstants.NO_SEQNUM);
// TODO: Log WARN if the regiondir does not exist in the fs. If its not
// there then something wonky about the split -- things will keep going
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/SplitRegionHandler.java Thu Feb 14 12:58:12 2013
@@ -46,6 +46,7 @@ public class SplitRegionHandler extends
/**
* For testing only! Set to true to skip handling of split.
*/
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="MS_SHOULD_BE_FINAL")
public static boolean TEST_SKIP = false;
public SplitRegionHandler(Server server,
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableAddFamilyHandler.java Thu Feb 14 12:58:12 2013
@@ -58,15 +58,13 @@ public class TableAddFamilyHandler exten
if(cpHost != null){
cpHost.preAddColumnHandler(this.tableName, this.familyDesc);
}
- // Update table descriptor in HDFS
- HTableDescriptor htd = this.masterServices.getMasterFileSystem()
- .addColumn(tableName, familyDesc);
- // Update in-memory descriptor cache
- this.masterServices.getTableDescriptors().add(htd);
+ // Update table descriptor
+ this.masterServices.getMasterFileSystem().addColumn(tableName, familyDesc);
if(cpHost != null){
cpHost.postAddColumnHandler(this.tableName, this.familyDesc);
}
}
+
@Override
public String toString() {
String name = "UnknownServerName";
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java Thu Feb 14 12:58:12 2013
@@ -53,11 +53,8 @@ public class TableDeleteFamilyHandler ex
if (cpHost != null) {
cpHost.preDeleteColumnHandler(this.tableName, this.familyName);
}
- // Update table descriptor in HDFS
- HTableDescriptor htd =
- this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName);
- // Update in-memory descriptor cache
- this.masterServices.getTableDescriptors().add(htd);
+ // Update table descriptor
+ this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName);
// Remove the column family from the file system
MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
for (HRegionInfo hri : hris) {
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java Thu Feb 14 12:58:12 2013
@@ -61,7 +61,6 @@ public abstract class TableEventHandler
protected final MasterServices masterServices;
protected final byte [] tableName;
protected final String tableNameStr;
- protected boolean persistedToZk = false;
public TableEventHandler(EventType eventType, byte [] tableName, Server server,
MasterServices masterServices)
@@ -111,10 +110,7 @@ public abstract class TableEventHandler
LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
} catch (KeeperException e) {
LOG.error("Error manipulating table " + Bytes.toString(tableName), e);
- } finally {
- // notify the waiting thread that we're done persisting the request
- setPersist();
- }
+ }
}
public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
@@ -165,29 +161,6 @@ public abstract class TableEventHandler
return done;
}
- /**
- * Table modifications are processed asynchronously, but provide an API for
- * you to query their status.
- *
- * @throws IOException
- */
- public synchronized void waitForPersist() throws IOException {
- if (!persistedToZk) {
- try {
- wait();
- } catch (InterruptedException ie) {
- throw (IOException) new InterruptedIOException().initCause(ie);
- }
- assert persistedToZk;
- }
- }
-
- private synchronized void setPersist() {
- if (!persistedToZk) {
- persistedToZk = true;
- notify();
- }
- }
/**
* Gets a TableDescriptor from the masterServices. Can Throw exceptions.
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableModifyFamilyHandler.java Thu Feb 14 12:58:12 2013
@@ -55,11 +55,8 @@ public class TableModifyFamilyHandler ex
if (cpHost != null) {
cpHost.preModifyColumnHandler(this.tableName, this.familyDesc);
}
- // Update table descriptor in HDFS
- HTableDescriptor htd =
- this.masterServices.getMasterFileSystem().modifyColumn(tableName, familyDesc);
- // Update in-memory descriptor cache
- this.masterServices.getTableDescriptors().add(htd);
+ // Update table descriptor
+ this.masterServices.getMasterFileSystem().modifyColumn(tableName, familyDesc);
if (cpHost != null) {
cpHost.postModifyColumnHandler(this.tableName, this.familyDesc);
}
Modified: hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java?rev=1446147&r1=1446146&r2=1446147&view=diff
==============================================================================
--- hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java (original)
+++ hbase/branches/hbase-7290v2/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/LogMonitoring.java Thu Feb 14 12:58:12 2013
@@ -79,17 +79,18 @@ public abstract class LogMonitoring {
private static void dumpTailOfLog(File f, PrintWriter out, long tailKb)
throws IOException {
FileInputStream fis = new FileInputStream(f);
+ BufferedReader r = null;
try {
FileChannel channel = fis.getChannel();
channel.position(Math.max(0, channel.size() - tailKb*1024));
- BufferedReader r = new BufferedReader(
- new InputStreamReader(fis));
+ r = new BufferedReader(new InputStreamReader(fis));
r.readLine(); // skip the first partial line
String line;
while ((line = r.readLine()) != null) {
out.println(line);
}
} finally {
+ if (r != null) IOUtils.closeStream(r);
IOUtils.closeStream(fis);
}
}