You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/08/01 00:08:07 UTC
[31/50] [abbrv] git commit: [HBASE-11379] Modified AlterTable to fix
stale .regioninfo and wrote tests
[HBASE-11379] Modified AlterTable to fix stale .regioninfo and wrote tests
Summary:
changed the alterTable() method in HMaster to rewrite .regioninfo files (with parallel threads) after modification
incorporated David's comments
Test Plan: created Tables, modified them, and then verified that the .regioninfo files were still correct
Reviewers: adela, daviddeng, elliott, manukranthk
Reviewed By: elliott, manukranthk
Subscribers: daviddeng, elliott, adela, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1395379
Tasks: 4490604
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43138 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75ed75d3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75ed75d3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75ed75d3
Branch: refs/heads/0.89-fb
Commit: 75ed75d3ed5746f35817863d0ead400231bfd9ac
Parents: a45454d
Author: yuq <yu...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 24 22:35:16 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HRegionInfo.java | 56 +++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 192 +++++-----
.../apache/hadoop/hbase/client/HBaseFsck.java | 21 +-
.../hadoop/hbase/ipc/HMasterInterface.java | 11 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 380 +++++++++++--------
.../hadoop/hbase/regionserver/HRegion.java | 16 +-
.../client/TestHBaseFsckFixRegionInfo.java | 102 +++--
.../hbase/master/TestAlterTableRegionInfo.java | 152 ++++++++
8 files changed, 584 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 9b30994..63701cb 100644
--- a/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -26,8 +26,14 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JenkinsHash;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.apache.hadoop.io.VersionedWritable;
@@ -115,6 +121,7 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
return encodedName;
}
+ public static final char NEWLINE = '\n';
/** delimiter used between portions of a region name */
public static final int DELIMITER = ',';
@@ -696,4 +703,53 @@ public class HRegionInfo extends VersionedWritable implements WritableComparable
return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
}
+
+ /**
+ * This method writes the .regioninfo file. This method is synchronized so that no two threads
+ * attempt to write the same .regioninfo file at the same time.
+ * @param conf
+ * @return true if the write was successful
+ * @throws IOException
+ */
+ public synchronized boolean writeToDisk(Configuration conf) throws IOException {
+ Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ FileSystem fs = rootDir.getFileSystem(conf);
+ return writeToDisk(conf, fs);
+ }
+
+ /**
+ * This method writes the .regioninfo file. This method is synchronized so that no two threads
+ * attempt to write the same .regioninfo file at the same time.
+ * @param conf
+ * @param fs
+ * @return true if the write is successful
+ * @throws IOException
+ */
+ public synchronized boolean writeToDisk(Configuration conf, FileSystem fs) throws IOException {
+ Path tableDir =
+ HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), this.getTableDesc().getName());
+ Path regionPath = HRegion.getRegionDir(tableDir, this.getEncodedName());
+ Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
+ FSDataOutputStream out = fs.create(regionInfoPath, true);
+ boolean successfulWrite = true;
+ try {
+ // we write to file the information necessary to reconstruct this HRegionInfo via readFields()
+ // this.toString() calls tableDesc.toString(), so this file will contain back-up information
+ // for the table
+ write(out);
+ out.write(NEWLINE);
+ out.write(NEWLINE);
+ out.write(Bytes.toBytes(this.toString()));
+ LOG.debug("Rewrote .regioninfo file of " + this.getRegionNameAsString() + " at "
+ + regionInfoPath);
+ } catch (IOException e) {
+ successfulWrite = false;
+ LOG.error("Could not rewrite the .regioninfo of " + this.getRegionNameAsString() + " at "
+ + regionInfoPath, e);
+ } finally {
+ out.close();
+ LOG.debug(".regioninfo File Contents: " + this.toString());
+ }
+ return successfulWrite;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index c16889b..f090cbc 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -140,7 +140,7 @@ public class HBaseAdmin {
* @throws MasterNotRunningException if the master is not running
*/
public boolean tableExists(final String tableName)
- throws MasterNotRunningException {
+ throws MasterNotRunningException {
return tableExists(new StringBytes(tableName));
}
@@ -150,7 +150,7 @@ public class HBaseAdmin {
* @throws MasterNotRunningException if the master is not running
*/
public boolean tableExists(final byte [] tableName)
- throws MasterNotRunningException {
+ throws MasterNotRunningException {
return tableExists(new StringBytes(tableName));
}
@@ -183,7 +183,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public HTableDescriptor getTableDescriptor(final byte [] tableName)
- throws IOException {
+ throws IOException {
return this.connection.getHTableDescriptor(new StringBytes(tableName));
}
@@ -208,7 +208,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void createTable(HTableDescriptor desc)
- throws IOException {
+ throws IOException {
createTable(desc, null);
}
@@ -238,7 +238,7 @@ public class HBaseAdmin {
*/
public void createTable(HTableDescriptor desc, byte [] startKey,
byte [] endKey, int numRegions)
- throws IOException {
+ throws IOException {
HTableDescriptor.isLegalTableName(desc.getName());
if(numRegions < 3) {
throw new IllegalArgumentException("Must create at least three regions");
@@ -270,7 +270,7 @@ public class HBaseAdmin {
* @throws IOException
*/
public void createTable(final HTableDescriptor desc, byte [][] splitKeys)
- throws IOException {
+ throws IOException {
HTableDescriptor.isLegalTableName(desc.getName());
checkSplitKeys(splitKeys);
createTableAsync(desc, splitKeys);
@@ -347,8 +347,8 @@ public class HBaseAdmin {
for (byte[] splitKey : splitKeys) {
if (Bytes.equals(splitKey, HConstants.EMPTY_BYTE_ARRAY)) {
throw new IllegalArgumentException(
- "Split keys cannot be empty"
- );
+ "Split keys cannot be empty"
+ );
}
if (lastKey != null && Bytes.equals(splitKey, lastKey)) {
throw new IllegalArgumentException(
@@ -373,7 +373,7 @@ public class HBaseAdmin {
* @throws IOException
*/
public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
- throws IOException {
+ throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
@@ -420,15 +420,15 @@ public class HBaseAdmin {
final int batchCount = this.conf.getInt("hbase.admin.scanner.caching", 10);
// Wait until first region is deleted
HRegionInterface server =
- connection.getHRegionConnection(firstMetaServer.getServerAddress());
+ connection.getHRegionConnection(firstMetaServer.getServerAddress());
HRegionInfo info = new HRegionInfo();
for (int tries = 0; tries < numRetries; tries++) {
long scannerId = -1L;
try {
Scan scan = new Scan().addColumn(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
+ HConstants.REGIONINFO_QUALIFIER);
scannerId = server.openScanner(
- firstMetaServer.getRegionInfo().getRegionName(), scan);
+ firstMetaServer.getRegionInfo().getRegionName(), scan);
// Get a batch at a time.
Result[] values = server.next(scannerId, batchCount);
if (values == null || values.length == 0) {
@@ -519,7 +519,7 @@ public class HBaseAdmin {
long sleep = getPauseTime(tries);
if (LOG.isDebugEnabled()) {
LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
- "enabled in " + Bytes.toStringBinary(tableName));
+ "enabled in " + Bytes.toStringBinary(tableName));
}
try {
Thread.sleep(sleep);
@@ -528,12 +528,12 @@ public class HBaseAdmin {
}
if (LOG.isDebugEnabled()) {
LOG.debug("Wake. Waiting for all regions to be enabled from " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
}
}
if (!enabled)
throw new IOException("Unable to enable table " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
LOG.info("Enabled table " + Bytes.toStringBinary(tableName));
}
@@ -574,7 +574,7 @@ public class HBaseAdmin {
if (disabled) break;
if (LOG.isDebugEnabled()) {
LOG.debug("Sleep. Waiting for all regions to be disabled from " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
}
try {
Thread.sleep(getPauseTime(tries));
@@ -583,12 +583,12 @@ public class HBaseAdmin {
}
if (LOG.isDebugEnabled()) {
LOG.debug("Wake. Waiting for all regions to be disabled from " +
- Bytes.toStringBinary(tableName));
+ Bytes.toStringBinary(tableName));
}
}
if (!disabled) {
throw new RegionException("Retries exhausted, it took too long to wait"+
- " for the table " + Bytes.toStringBinary(tableName) + " to be disabled.");
+ " for the table " + Bytes.toStringBinary(tableName) + " to be disabled.");
}
LOG.info("Disabled " + Bytes.toStringBinary(tableName));
}
@@ -649,15 +649,18 @@ public class HBaseAdmin {
/**
* Batch alter a table. Only takes regions offline once and performs a single
* update to .META.
+ * Rewrite the .regioninfo files that belong to the modified table
* Asynchronous operation.
*
* @param tableName name of the table to add column to
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files that belong to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final String tableName,
+ public boolean alterTable(final String tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<String, HColumnDescriptor>> columnModifications,
List<String> columnDeletions) throws IOException {
@@ -669,7 +672,7 @@ public class HBaseAdmin {
int maxClosedRegions = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE);
- alterTable(tableName, columnAdditions, columnModifications,
+ return alterTable(tableName, columnAdditions, columnModifications,
columnDeletions, waitInterval, maxClosedRegions);
}
@@ -686,14 +689,16 @@ public class HBaseAdmin {
* numConcurrentRegionsClosed over
* @param maxConcurrentRegionsClosed the max number of regions to have closed
* at a time.
+ * @return true if all .regioninfo files belonging to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final String tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<String, HColumnDescriptor>> columnModifications,
- List<String> columnDeletions,
- int waitInterval,
- int maxConcurrentRegionsClosed) throws IOException {
+ public boolean alterTable(final String tableName,
+ List<HColumnDescriptor> columnAdditions,
+ List<Pair<String, HColumnDescriptor>> columnModifications,
+ List<String> columnDeletions,
+ int waitInterval,
+ int maxConcurrentRegionsClosed) throws IOException {
// convert all of the strings to bytes and pass to the bytes method
List<Pair<byte [], HColumnDescriptor>> modificationsBytes =
new ArrayList<Pair<byte [], HColumnDescriptor>>(
@@ -701,21 +706,22 @@ public class HBaseAdmin {
List<byte []> deletionsBytes =
new ArrayList<byte []>(columnDeletions.size());
- for(Pair<String, HColumnDescriptor> c : columnModifications) {
- modificationsBytes.add(new Pair<byte [], HColumnDescriptor>(
- Bytes.toBytes(c.getFirst()), c.getSecond()));
- }
- for(String c : columnDeletions) {
- deletionsBytes.add(Bytes.toBytes(c));
- }
+ for(Pair<String, HColumnDescriptor> c : columnModifications) {
+ modificationsBytes.add(new Pair<byte [], HColumnDescriptor>(
+ Bytes.toBytes(c.getFirst()), c.getSecond()));
+ }
+ for(String c : columnDeletions) {
+ deletionsBytes.add(Bytes.toBytes(c));
+ }
- alterTable(Bytes.toBytes(tableName), columnAdditions, modificationsBytes,
- deletionsBytes, waitInterval, maxConcurrentRegionsClosed);
+ return alterTable(Bytes.toBytes(tableName), columnAdditions, modificationsBytes,
+ deletionsBytes, waitInterval, maxConcurrentRegionsClosed);
}
/**
* Batch alter a table. Only takes regions offline once and performs a single
* update to .META.
+ * Rewrite the .regioninfo files that belong to the modified table
* Any of the three lists can be null, in which case those types of
* alterations will be ignored.
* Asynchronous operation.
@@ -724,9 +730,11 @@ public class HBaseAdmin {
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files belonging to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final byte [] tableName,
+ public boolean alterTable(final byte [] tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<byte[], HColumnDescriptor>> columnModifications,
List<byte[]> columnDeletions) throws IOException {
@@ -739,12 +747,13 @@ public class HBaseAdmin {
int maxClosedRegions = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE);
- alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
+ return alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
}
/**
* Batch alter a table. Only takes regions offline once and performs a single
* update to .META.
+ * Rewrite all the .regioninfo files that belong to the modified table
* Any of the three lists can be null, in which case those types of
* alterations will be ignored.
* Asynchronous operation.
@@ -757,25 +766,27 @@ public class HBaseAdmin {
* numConcurrentRegionsClosed over
* @param maxConcurrentRegionsClosed the max number of regions to have closed
* at a time.
+ * @return true if all .regioninfo files belonging to the modified table
+ * are successfully rewritten
* @throws IOException if a remote or network exception occurs
*/
- public void alterTable(final byte [] tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<byte[], HColumnDescriptor>> columnModifications,
- List<byte[]> columnDeletions,
- int waitInterval,
- int maxConcurrentRegionsClosed) throws IOException {
+ public boolean alterTable(final byte [] tableName,
+ List<HColumnDescriptor> columnAdditions,
+ List<Pair<byte[], HColumnDescriptor>> columnModifications,
+ List<byte[]> columnDeletions,
+ int waitInterval,
+ int maxConcurrentRegionsClosed) throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
HTableDescriptor.isLegalTableName(tableName);
try {
- this.master.alterTable(tableName, columnAdditions, columnModifications,
+ return this.master.alterTable(tableName, columnAdditions, columnModifications,
columnDeletions, waitInterval, maxConcurrentRegionsClosed);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
- }
+}
/**
* Get the status of alter command - indicates how many regions have received
@@ -808,7 +819,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void addColumn(final String tableName, HColumnDescriptor column)
- throws IOException {
+ throws IOException {
alterTable(Bytes.toBytes(tableName), Arrays.asList(column), null, null);
}
@@ -821,7 +832,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void addColumn(final byte [] tableName, HColumnDescriptor column)
- throws IOException {
+ throws IOException {
alterTable(tableName, Arrays.asList(column), null, null);
}
@@ -834,7 +845,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void deleteColumn(final String tableName, final String columnName)
- throws IOException {
+ throws IOException {
alterTable(Bytes.toBytes(tableName), null, null,
Arrays.asList(Bytes.toBytes(columnName)));
}
@@ -849,7 +860,7 @@ public class HBaseAdmin {
*/
public void deleteColumn(final byte [] tableName,
final byte [] columnName)
- throws IOException {
+ throws IOException {
alterTable(tableName, null, null, Arrays.asList(columnName));
}
@@ -863,15 +874,15 @@ public class HBaseAdmin {
*/
public void modifyColumn(final String tableName, final String columnName,
HColumnDescriptor descriptor)
- throws IOException {
+ throws IOException {
alterTable(Bytes.toBytes(tableName), null, Arrays.asList(
- new Pair<byte [], HColumnDescriptor>(Bytes.toBytes(columnName),
+ new Pair<byte [], HColumnDescriptor>(Bytes.toBytes(columnName),
descriptor)), null);
}
public void modifyColumn(final String tableName,
HColumnDescriptor descriptor)
- throws IOException {
+ throws IOException {
modifyColumn(tableName, descriptor.getNameAsString(), descriptor);
}
@@ -885,10 +896,10 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void modifyColumn(final byte [] tableName, final byte [] columnName,
- HColumnDescriptor descriptor)
- throws IOException {
+ HColumnDescriptor descriptor)
+ throws IOException {
alterTable(tableName, null, Arrays.asList(
- new Pair<byte [], HColumnDescriptor>(columnName, descriptor)), null);
+ new Pair<byte [], HColumnDescriptor>(columnName, descriptor)), null);
}
/**
@@ -901,7 +912,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void closeRegion(final String regionname, final Object... args)
- throws IOException {
+ throws IOException {
closeRegion(Bytes.toBytes(regionname), args);
}
@@ -915,7 +926,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void closeRegion(final byte [] regionname, final Object... args)
- throws IOException {
+ throws IOException {
// Be careful. Must match the handler over in HMaster at MODIFY_CLOSE_REGION
int len = (args == null)? 0: args.length;
int xtraArgsCount = 1;
@@ -925,7 +936,7 @@ public class HBaseAdmin {
System.arraycopy(args, 0, newargs, xtraArgsCount, len);
}
modifyTable(HConstants.META_TABLE_NAME, HConstants.Modify.CLOSE_REGION,
- newargs);
+ newargs);
}
/**
@@ -939,7 +950,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void moveRegion(final String regionName, final String regionServer)
- throws IOException {
+ throws IOException {
moveRegion(Bytes.toBytes(regionName), regionServer);
}
@@ -954,9 +965,9 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void moveRegion(final byte[] regionName, final String regionServer)
- throws IOException {
+ throws IOException {
modifyTable(HConstants.META_TABLE_NAME, HConstants.Modify.MOVE_REGION,
- new Object[]{regionName, regionServer});
+ new Object[]{regionName, regionServer});
}
/**
@@ -1012,7 +1023,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
private void compactCF(String tableName, String columnFamily, HConstants.Modify op)
- throws IOException {
+ throws IOException {
compactCF(Bytes.toBytes(tableName), Bytes.toBytes(columnFamily), op);
}
@@ -1025,11 +1036,11 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
private void compactCF(final byte[] tableName, final byte[] columnFamily, HConstants.Modify op)
- throws IOException {
+ throws IOException {
// Validate table name and column family.
if (!this.connection.tableExists(new StringBytes(tableName))) {
throw new IllegalArgumentException("HTable " + new StringBytes(tableName)
- + " does not exist");
+ + " does not exist");
} else if (!getTableDescriptor(tableName).hasFamily(columnFamily)) {
throw new IllegalArgumentException("Column Family "
+ new String(columnFamily) + " does not exist in "
@@ -1040,6 +1051,7 @@ public class HBaseAdmin {
HTable table = new HTable(this.conf, tableName);
Set <HRegionInfo> regions = table.getRegionsInfo().keySet();
Iterator <HRegionInfo> regionsIt = regions.iterator();
+ table.close();
// Iterate over all regions and send a compaction request to each.
while (regionsIt.hasNext()) {
@@ -1057,7 +1069,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void compact(String tableOrRegionName, String columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily, HConstants.Modify.TABLE_COMPACT);
return;
@@ -1080,7 +1092,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void compact(final byte[] tableOrRegionName, final byte[] columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily, HConstants.Modify.TABLE_COMPACT);
return;
@@ -1100,7 +1112,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(final String tableNameOrRegionName)
- throws IOException {
+ throws IOException {
majorCompact(Bytes.toBytes(tableNameOrRegionName));
}
@@ -1112,7 +1124,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(final byte [] tableNameOrRegionName)
- throws IOException {
+ throws IOException {
modifyTable(tableNameOrRegionName, HConstants.Modify.TABLE_MAJOR_COMPACT);
}
@@ -1125,7 +1137,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(String tableOrRegionName, String columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily,
HConstants.Modify.TABLE_MAJOR_COMPACT);
@@ -1148,7 +1160,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void majorCompact(final byte[] tableOrRegionName, final byte[] columnFamily)
- throws IOException {
+ throws IOException {
if (tableExists(tableOrRegionName)) {
compactCF(tableOrRegionName, columnFamily,
HConstants.Modify.TABLE_MAJOR_COMPACT);
@@ -1181,7 +1193,7 @@ public class HBaseAdmin {
}
public void split(final String tableNameOrRegionName,
- final String splitPoint) throws IOException {
+ final String splitPoint) throws IOException {
split(Bytes.toBytes(tableNameOrRegionName), Bytes.toBytes(splitPoint));
}
@@ -1199,11 +1211,11 @@ public class HBaseAdmin {
throw new IllegalArgumentException("Pass a table name or region name");
}
byte [] tableName = tableExists(tableNameOrRegionName)?
- tableNameOrRegionName: null;
+ tableNameOrRegionName: null;
byte [] regionName = tableName == null? tableNameOrRegionName: null;
Object [] args = regionName == null?
- new byte [][] {splitPoint}: new byte [][] {regionName, splitPoint};
- modifyTable(tableName, HConstants.Modify.TABLE_EXPLICIT_SPLIT, args);
+ new byte [][] {splitPoint}: new byte [][] {regionName, splitPoint};
+ modifyTable(tableName, HConstants.Modify.TABLE_EXPLICIT_SPLIT, args);
}
/*
@@ -1215,7 +1227,7 @@ public class HBaseAdmin {
*/
private void modifyTable(final byte [] tableNameOrRegionName,
final HConstants.Modify op)
- throws IOException {
+ throws IOException {
if (tableNameOrRegionName == null) {
throw new IllegalArgumentException("Pass a table name or region name");
}
@@ -1236,7 +1248,7 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void modifyTable(final byte [] tableName, HTableDescriptor htd)
- throws IOException {
+ throws IOException {
modifyTable(tableName, HConstants.Modify.TABLE_SET_HTD, htd);
}
@@ -1252,7 +1264,7 @@ public class HBaseAdmin {
*/
public void modifyTable(final byte [] tableName, HConstants.Modify op,
Object... args)
- throws IOException {
+ throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
@@ -1266,7 +1278,7 @@ public class HBaseAdmin {
switch (op) {
case TABLE_SET_HTD:
if (args == null || args.length < 1 ||
- !(args[0] instanceof HTableDescriptor)) {
+ !(args[0] instanceof HTableDescriptor)) {
throw new IllegalArgumentException("SET_HTD requires a HTableDescriptor");
}
arr = new Writable[1];
@@ -1325,7 +1337,7 @@ public class HBaseAdmin {
return new BooleanWritable((Boolean) o);
} else {
throw new IllegalArgumentException("Requires byte [] or " +
- "ImmutableBytesWritable, not " + o.getClass() + " : " + o);
+ "ImmutableBytesWritable, not " + o.getClass() + " : " + o);
}
}
@@ -1358,8 +1370,8 @@ public class HBaseAdmin {
public synchronized void stopRegionServerForRestart(final HServerAddress hsa)
throws IOException {
HRegionInterface rs = this.connection.getHRegionConnection(hsa);
- LOG.info("Restarting RegionServer" + hsa.toString());
- rs.stopForRestart();
+ LOG.info("Restarting RegionServer" + hsa.toString());
+ rs.stopForRestart();
}
/**
@@ -1393,9 +1405,9 @@ public class HBaseAdmin {
public synchronized void setNumHDFSQuorumReadThreads(final HServerAddress hsa,
int numThreads) throws IOException {
HRegionInterface rs = this.connection.getHRegionConnection(hsa);
- LOG.info("Setting numHDFSQuorumReadThreads for RegionServer" + hsa.toString()
- + " to " + numThreads);
- rs.setNumHDFSQuorumReadThreads(numThreads);
+ LOG.info("Setting numHDFSQuorumReadThreads for RegionServer" + hsa.toString()
+ + " to " + numThreads);
+ rs.setNumHDFSQuorumReadThreads(numThreads);
}
/**
@@ -1409,9 +1421,9 @@ public class HBaseAdmin {
public synchronized void setHDFSQuorumReadTimeoutMillis(final HServerAddress hsa,
long timeoutMillis) throws IOException {
HRegionInterface rs = this.connection.getHRegionConnection(hsa);
- LOG.info("Setting quorumReadTimeout for RegionServer" + hsa.toString()
- + " to " + timeoutMillis);
- rs.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
+ LOG.info("Setting quorumReadTimeout for RegionServer" + hsa.toString()
+ + " to " + timeoutMillis);
+ rs.setHDFSQuorumReadTimeoutMillis(timeoutMillis);
}
/**
@@ -1427,9 +1439,9 @@ public class HBaseAdmin {
}
private HRegionLocation getFirstMetaServerForTable(final byte [] tableName)
- throws IOException {
+ throws IOException {
return connection.locateRegion(HConstants.META_TABLE_NAME_STRINGBYTES,
- HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
+ HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false));
}
/**
@@ -1439,7 +1451,7 @@ public class HBaseAdmin {
* @throws MasterNotRunningException if a remote or network exception occurs
*/
public static void checkHBaseAvailable(Configuration conf)
- throws MasterNotRunningException {
+ throws MasterNotRunningException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
copyOfConf.setInt(HConstants.CLIENT_RETRY_NUM_STRING, 1);
new HBaseAdmin(copyOfConf);
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
index 5206f4a..6228151 100644
--- a/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
+++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
@@ -46,7 +46,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -328,27 +327,9 @@ public class HBaseFsck {
void fixRegionInfo() throws IOException {
Map<HRegionInfo, Path> risToRewrite = checkRegionInfo();
LOG.info("FOUND " + risToRewrite.size() + " corrupted .regioninfo files to rewrite");
- Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
- FileSystem fs = rootDir.getFileSystem(conf);
for(Map.Entry<HRegionInfo, Path> entry : risToRewrite.entrySet()){
HRegionInfo hri = entry.getKey();
- Path regionInfoPath = entry.getValue();
- FSDataOutputStream out = fs.create(regionInfoPath, true);
-
- try {
- hri.write(out);
- out.write('\n');
- out.write('\n');
- out.write(Bytes.toBytes(hri.toString()));
- LOG.debug("Rewrote .regioninfo file of " + Bytes.toString(hri.getRegionName()) +"at " + regionInfoPath);
- } catch (IOException e){
- errors.reportError("Could not rewrite the .regioninfo of "+Bytes.toString(hri.getRegionName()) + " at " +regionInfoPath);
- }
-
- finally {
- out.close();
- LOG.debug(".regioninfo File Contents: " + Bytes.toBytes(hri.toString()));
- }
+ hri.writeToDisk(conf);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
index c09f117..5a9bb5b 100644
--- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
+++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.Writable;
@@ -69,14 +68,17 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion, ThriftClientI
* Batch adds, modifies, and deletes columns from the specified table.
* Any of the lists may be null, in which case those types of alterations
* will not occur.
+ * .regioninfo files in the relevant table will be properly rewritten after
+ * the modification
*
* @param tableName table to modify
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files in the table were properly rewritten
* @throws IOException e
*/
- public void alterTable(final byte [] tableName,
+ public boolean alterTable(final byte [] tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<byte [], HColumnDescriptor>> columnModifications,
List<byte []> columnDeletions) throws IOException;
@@ -85,14 +87,17 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion, ThriftClientI
* Batch adds, modifies, and deletes columns from the specified table.
* Any of the lists may be null, in which case those types of alterations
* will not occur.
+ * .regioninfo files in the relevant table will be properly rewritten after
+ * the modification
*
* @param tableName table to modify
* @param columnAdditions column descriptors to add to the table
* @param columnModifications pairs of column names with new descriptors
* @param columnDeletions column names to delete from the table
+ * @return true if all .regioninfo files in the table were properly rewritten
* @throws IOException e
*/
- public void alterTable(final byte [] tableName,
+ public boolean alterTable(final byte [] tableName,
List<HColumnDescriptor> columnAdditions,
List<Pair<byte [], HColumnDescriptor>> columnModifications,
List<byte []> columnDeletions,
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c95f821..9b51280 100755
--- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -40,7 +40,11 @@ import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Set;
import java.util.SortedMap;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -142,7 +146,7 @@ import com.google.common.collect.Lists;
* @see Watcher
*/
public class HMaster extends HasThread implements HMasterInterface,
- HMasterRegionInterface, Watcher, StoppableMaster {
+HMasterRegionInterface, Watcher, StoppableMaster {
// MASTER is name of the webapp and the attribute name used stuffing this
//instance into web context.
public static final String MASTER = "master";
@@ -361,7 +365,7 @@ public class HMaster extends HasThread implements HMasterInterface,
LOG.debug("HMaster started in backup mode. Stall " + stallTime +
"ms giving primary master a fair chance to be the master...");
try {
- Thread.sleep(stallTime);
+ Thread.sleep(stallTime);
} catch (InterruptedException e) {
// interrupted = user wants to kill us. Don't continue
throw new IOException("Interrupted waiting for master address");
@@ -371,7 +375,7 @@ public class HMaster extends HasThread implements HMasterInterface,
final String masterName = getServerName();
// initialize the thread pool for non-distributed log splitting.
int maxSplitLogThread =
- conf.getInt("hbase.master.splitLogThread.max", 1000);
+ conf.getInt("hbase.master.splitLogThread.max", 1000);
logSplitThreadPool = Threads.getBoundedCachedThreadPool(
maxSplitLogThread, 30L, TimeUnit.SECONDS,
new ThreadFactory() {
@@ -391,18 +395,18 @@ public class HMaster extends HasThread implements HMasterInterface,
// Only read favored nodes if using the assignment-based load balancer.
this.shouldAssignRegionsWithFavoredNodes = conf.getClass(
HConstants.LOAD_BALANCER_IMPL, Object.class).equals(
- RegionManager.AssignmentLoadBalancer.class);
+ RegionManager.AssignmentLoadBalancer.class);
LOG.debug("Whether to read the favoredNodes from meta: " +
(shouldAssignRegionsWithFavoredNodes ? "Yes" : "No"));
// Initialize table level lock manager for schema changes, if enabled.
if (conf.getBoolean(HConstants.MASTER_SCHEMA_CHANGES_LOCK_ENABLE,
- HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_ENABLE)) {
+ HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_ENABLE)) {
int schemaChangeLockTimeoutMs = conf.getInt(
- HConstants.MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS,
- HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS);
+ HConstants.MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS,
+ HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_LOCK_TIMEOUT_MS);
tableLockManager = new TableLockManager(zooKeeperWrapper,
- address, schemaChangeLockTimeoutMs);
+ address, schemaChangeLockTimeoutMs);
this.schemaChangeTryLockTimeoutMs = conf.getInt(
HConstants.MASTER_SCHEMA_CHANGES_TRY_LOCK_TIMEOUT_MS,
@@ -477,8 +481,8 @@ public class HMaster extends HasThread implements HMasterInterface,
this.getConfigurationManager().registerObserver(serverManager);
this.regionServerOperationQueue =
- new RegionServerOperationQueue(this.conf, serverManager,
- getClosedStatus());
+ new RegionServerOperationQueue(this.conf, serverManager,
+ getClosedStatus());
// start the "close region" executor service
HBaseEventType.RS2ZK_REGION_CLOSED.startMasterExecutorService(
@@ -539,16 +543,16 @@ public class HMaster extends HasThread implements HMasterInterface,
zooKeeperWrapper = ZooKeeperWrapper.createInstance(localConf,
getZKWrapperName(), new Abortable() {
- @Override
- public void abort(String why, Throwable e) {
- stop("ZK session expired");
- }
+ @Override
+ public void abort(String why, Throwable e) {
+ stop("ZK session expired");
+ }
- @Override
- public boolean isAborted() {
- return stopped;
- }
- });
+ @Override
+ public boolean isAborted() {
+ return stopped;
+ }
+ });
}
}
@@ -595,8 +599,8 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
private static Path checkRootDir(final Path rd, final Configuration c,
- final FileSystem fs)
- throws IOException {
+ final FileSystem fs)
+ throws IOException {
// If FS is in safe mode wait till out of it.
FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
10 * 1000));
@@ -615,7 +619,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
private static void bootstrap(final Path rd, final Configuration c)
- throws IOException {
+ throws IOException {
LOG.info("BOOTSTRAP: creating ROOT and first META regions");
try {
// Bootstrapping, make sure blockcache is off. Else, one will be
@@ -623,21 +627,21 @@ public class HMaster extends HasThread implements HMasterInterface,
// not make it in first place. Turn off block caching for bootstrap.
// Enable after.
HRegionInfo rootHRI = HTableDescriptor.isMetaregionSeqidRecordEnabled(c) ?
- new HRegionInfo(HRegionInfo.ROOT_REGIONINFO_WITH_HISTORIAN_COLUMN) :
- new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
- setInfoFamilyCaching(rootHRI, false);
- HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
- setInfoFamilyCaching(metaHRI, false);
- HRegion root = HRegion.createHRegion(rootHRI, rd, c);
- HRegion meta = HRegion.createHRegion(metaHRI, rd, c);
- setInfoFamilyCaching(rootHRI, true);
- setInfoFamilyCaching(metaHRI, true);
- // Add first region from the META table to the ROOT region.
- HRegion.addRegionToMETA(root, meta);
- root.close();
- root.getLog().closeAndDelete();
- meta.close();
- meta.getLog().closeAndDelete();
+ new HRegionInfo(HRegionInfo.ROOT_REGIONINFO_WITH_HISTORIAN_COLUMN) :
+ new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
+ setInfoFamilyCaching(rootHRI, false);
+ HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
+ setInfoFamilyCaching(metaHRI, false);
+ HRegion root = HRegion.createHRegion(rootHRI, rd, c);
+ HRegion meta = HRegion.createHRegion(metaHRI, rd, c);
+ setInfoFamilyCaching(rootHRI, true);
+ setInfoFamilyCaching(metaHRI, true);
+ // Add first region from the META table to the ROOT region.
+ HRegion.addRegionToMETA(root, meta);
+ root.close();
+ root.getLog().closeAndDelete();
+ meta.close();
+ meta.getLog().closeAndDelete();
} catch (IOException e) {
e = RemoteExceptionHandler.checkIOException(e);
LOG.error("bootstrap", e);
@@ -663,10 +667,10 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws UnknownHostException
*/
private static String getMyAddress(final Configuration c)
- throws UnknownHostException, SocketException {
+ throws UnknownHostException, SocketException {
// Find out our address up in DNS.
String s = DNS.getDefaultHost(c.get("hbase.master.dns.interface","default"),
- c.get("hbase.master.dns.nameserver","default"));
+ c.get("hbase.master.dns.nameserver","default"));
if (preferIpv6AddressForMaster(c)) {
// Use IPv6 address if possible.
s = HServerInfo.getIPv6AddrIfLocalMachine(s);
@@ -818,7 +822,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
MonitoredTask startupStatus =
- TaskMonitor.get().createStatus("Master startup");
+ TaskMonitor.get().createStatus("Master startup");
startupStatus.setDescription("Master startup");
clusterStateRecovery = new ZKClusterStateRecovery(this, connection);
try {
@@ -887,19 +891,19 @@ public class HMaster extends HasThread implements HMasterInterface,
break;
} else {
LOG.debug("Waiting on " +
- this.serverManager.getServersToServerInfo().keySet().toString());
+ this.serverManager.getServersToServerInfo().keySet().toString());
}
}
switch (this.regionServerOperationQueue.process()) {
case FAILED:
- // If FAILED op processing, bad. Exit.
+ // If FAILED op processing, bad. Exit.
break FINISHED;
case REQUEUED_BUT_PROBLEM:
// LOG if the file system is down, but don't do anything.
checkFileSystem(false);
break;
default:
- // Continue run loop if conditions are PROCESSED, NOOP, REQUEUED
+ // Continue run loop if conditions are PROCESSED, NOOP, REQUEUED
break;
}
}
@@ -944,8 +948,8 @@ public class HMaster extends HasThread implements HMasterInterface,
private void initPreferredAssignment() {
// assign the regions based on the region locality in this period of time
this.applyPreferredAssignmentPeriod =
- conf.getLong("hbase.master.applyPreferredAssignment.period",
- 5 * 60 * 1000);
+ conf.getLong("hbase.master.applyPreferredAssignment.period",
+ 5 * 60 * 1000);
// disable scanning dfs by setting applyPreferredAssignmentPeriod to 0
if (applyPreferredAssignmentPeriod > 0) {
@@ -953,8 +957,8 @@ public class HMaster extends HasThread implements HMasterInterface,
// since master startup, then the master is free to assign this region
// out to any region server
this.holdRegionForBestLocalityPeriod =
- conf.getLong("hbase.master.holdRegionForBestLocality.period",
- 1 * 60 * 1000);
+ conf.getLong("hbase.master.holdRegionForBestLocality.period",
+ 1 * 60 * 1000);
// try to get the locality map from disk
this.preferredRegionToRegionServerMapping = getRegionLocalityFromSnapshot(conf);
@@ -969,24 +973,24 @@ public class HMaster extends HasThread implements HMasterInterface,
public static MapWritable getRegionLocalityFromSnapshot(Configuration conf) {
String region_assignment_snapshot_dir =
- conf.get("hbase.tmp.dir");
+ conf.get("hbase.tmp.dir");
if (region_assignment_snapshot_dir == null) {
return null;
}
String region_assignment_snapshot =
- region_assignment_snapshot_dir + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
+ region_assignment_snapshot_dir + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
long refresh_interval =
- conf.getLong("hbase.master.regionLocality.snapshot.validity_time_ms",
- 24 * 60 * 60 * 1000);
+ conf.getLong("hbase.master.regionLocality.snapshot.validity_time_ms",
+ 24 * 60 * 60 * 1000);
File snapshotFile = new File(region_assignment_snapshot);
try {
if (!snapshotFile.exists()) {
- LOG.info("preferredRegionToRegionServerMapping snapshot not found. File Path: "
- + region_assignment_snapshot);
- return null;
+ LOG.info("preferredRegionToRegionServerMapping snapshot not found. File Path: "
+ + region_assignment_snapshot);
+ return null;
}
long time_elapsed = System.currentTimeMillis() - snapshotFile.lastModified();
@@ -1028,10 +1032,10 @@ public class HMaster extends HasThread implements HMasterInterface,
LOG.debug("Evaluate preferredRegionToRegionServerMapping; expecting pause here");
try {
regionLocalityMap = FSUtils
- .getRegionLocalityMappingFromFS(FileSystem.get(conf), FSUtils.getRootDir(conf),
- poolSize,
- conf,
- tablename);
+ .getRegionLocalityMappingFromFS(FileSystem.get(conf), FSUtils.getRootDir(conf),
+ poolSize,
+ conf,
+ tablename);
} catch (Exception e) {
LOG.error("Got unexpected exception when evaluating " +
"preferredRegionToRegionServerMapping : " + e.toString());
@@ -1047,7 +1051,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
String region_assignment_snapshot = tmp_path
- + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
+ + "/" + LOCALITY_SNAPSHOT_FILE_NAME;
// write the preferredRegionAssignment to disk
try {
LOG.info("Saving preferredRegionToRegionServerMapping " +
@@ -1055,8 +1059,8 @@ public class HMaster extends HasThread implements HMasterInterface,
regionLocalityMap.write(new DataOutputStream(
new FileOutputStream(region_assignment_snapshot)));
} catch (IOException e) {
- LOG.error("Error saving preferredRegionToRegionServerMapping " +
- "to file " + region_assignment_snapshot + " : " + e.toString());
+ LOG.error("Error saving preferredRegionToRegionServerMapping " +
+ "to file " + region_assignment_snapshot + " : " + e.toString());
}
return regionLocalityMap;
}
@@ -1298,18 +1302,18 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public MapWritable regionServerStartup(final HServerInfo serverInfo)
- throws IOException {
+ throws IOException {
// Set the ip into the passed in serverInfo. Its ip is more than likely
// not the ip that the master sees here. See at end of this method where
// we pass it back to the regionserver by setting "hbase.regionserver.address"
String rsAddress = HBaseServer.getRemoteAddress();
serverInfo.setServerAddress(new HServerAddress(rsAddress,
- serverInfo.getServerAddress().getPort()));
+ serverInfo.getServerAddress().getPort()));
// Register with server manager
this.serverManager.regionServerStartup(serverInfo);
// Send back some config info
MapWritable mw = createConfigurationSubset();
- mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
+ mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
return mw;
}
@@ -1329,10 +1333,10 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public HMsg [] regionServerReport(HServerInfo serverInfo, HMsg msgs[],
- HRegionInfo[] mostLoadedRegions)
- throws IOException {
+ HRegionInfo[] mostLoadedRegions)
+ throws IOException {
return adornRegionServerAnswer(serverInfo,
- this.serverManager.regionServerReport(serverInfo, msgs, mostLoadedRegions));
+ this.serverManager.regionServerReport(serverInfo, msgs, mostLoadedRegions));
}
void updateLastFlushedSequenceIds(HServerInfo serverInfo) {
@@ -1349,7 +1353,7 @@ public class HMaster extends HasThread implements HMasterInterface,
+ Bytes.toStringBinary(entry.getKey()) + " Ignoring.");
}
continue; // Don't let smaller sequence ids override greater
- // sequence ids.
+ // sequence ids.
}
}
flushedSequenceIdByRegion.put(entry.getKey(), entry.getValue());
@@ -1429,14 +1433,14 @@ public class HMaster extends HasThread implements HMasterInterface,
}
protected void lockTable(byte[] tableName, String purpose)
- throws IOException {
+ throws IOException {
if (isTableLockEnabled()) {
tableLockManager.lockTable(tableName, purpose);
}
}
protected void unlockTable(byte[] tableName)
- throws IOException {
+ throws IOException {
if (isTableLockEnabled()) {
tableLockManager.unlockTable(tableName);
}
@@ -1475,7 +1479,7 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public void createTable(HTableDescriptor desc, byte [][] splitKeys)
- throws IOException {
+ throws IOException {
HRegionInfo[] newRegions = createRegionsForNewTable(desc, splitKeys);
try {
// We can not create a table unless meta regions have already been
@@ -1492,14 +1496,14 @@ public class HMaster extends HasThread implements HMasterInterface,
throw e;
} catch (IOException e) {
LOG.error("Cannot create table " + desc.getNameAsString() +
- " because of " + e.toString());
+ " because of " + e.toString());
throw RemoteExceptionHandler.checkIOException(e);
}
}
private static boolean tableExists(HRegionInterface srvr,
byte[] metaRegionName, String tableName)
- throws IOException {
+ throws IOException {
byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
Scan scan = new Scan(firstRowInTable);
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
@@ -1509,8 +1513,8 @@ public class HMaster extends HasThread implements HMasterInterface,
BaseScanner.getOneResultFromScanner(srvr, scannerid);
if (data != null && data.size() > 0) {
HRegionInfo info = Writables.getHRegionInfo(
- data.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER));
+ data.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
if (info.getTableDesc().getNameAsString().equals(tableName)) {
// A region for this table already exists. Ergo table exists.
return true;
@@ -1569,7 +1573,7 @@ public class HMaster extends HasThread implements HMasterInterface,
if (assignmentPlan != null) {
// create the region with favorite nodes.
List<HServerAddress> favoredNodes =
- assignmentPlan.getAssignment(newRegion);
+ assignmentPlan.getAssignment(newRegion);
regionManager.createRegion(newRegion, srvr, metaRegionName,
favoredNodes);
} else {
@@ -1611,33 +1615,47 @@ public class HMaster extends HasThread implements HMasterInterface,
}
@Override
- public void alterTable(final byte [] tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<byte [], HColumnDescriptor>> columnModifications,
- List<byte []> columnDeletions,
- int waitInterval,
- int maxConcurrentRegionsClosed) throws IOException {
- //This lock will be released when the ThrottledRegionReopener is done.
+ /**
+ * returns true if all .regioninfo files were succesfully rewritten after alteration
+ */
+ public boolean alterTable(final byte[] tableName, List<HColumnDescriptor> columnAdditions,
+ List<Pair<byte[], HColumnDescriptor>> columnModifications, List<byte[]> columnDeletions,
+ int waitInterval, int maxConcurrentRegionsClosed) throws IOException {
+ // This lock will be released when the ThrottledRegionReopener is done.
if (!tryLockTable(tableName, "alter", schemaChangeTryLockTimeoutMs)) {
throw new TableLockTimeoutException("Timed out acquiring lock for "
- + Bytes.toStringBinary(tableName) + " after "
- + schemaChangeTryLockTimeoutMs + " ms.");
+ + Bytes.toStringBinary(tableName) + " after " + schemaChangeTryLockTimeoutMs + " ms.");
}
InjectionHandler.processEvent(InjectionEvent.HMASTER_ALTER_TABLE);
- ThrottledRegionReopener reopener = this.regionManager.
- createThrottledReopener(Bytes.toString(tableName), waitInterval, maxConcurrentRegionsClosed);
+ ThrottledRegionReopener reopener =
+ this.regionManager.createThrottledReopener(Bytes.toString(tableName), waitInterval,
+ maxConcurrentRegionsClosed);
// Regions are added to the reopener in MultiColumnOperation
- new MultiColumnOperation(this, tableName, columnAdditions,
- columnModifications, columnDeletions).process();
+ new MultiColumnOperation(this, tableName, columnAdditions, columnModifications, columnDeletions)
+ .process();
reopener.startRegionsReopening();
+
+ // after the alteration, all .regioninfo files for this table will be old so we rewrite them.
+ try {
+ writeRegionInfo(tableName);
+ } catch (Exception e) {
+ LOG.error(
+ "Failed to complete rewriting .regioninfo files for table "
+ + Bytes.toStringBinary(tableName), e);
+ return false;
+ }
+ return true;
}
@Override
- public void alterTable(final byte [] tableName,
- List<HColumnDescriptor> columnAdditions,
- List<Pair<byte [], HColumnDescriptor>> columnModifications,
- List<byte []> columnDeletions) throws IOException {
+ /**
+ * Returns true if all backup .regioninfo files were successfully rewritten
+ */
+ public boolean alterTable(final byte [] tableName,
+ List<HColumnDescriptor> columnAdditions,
+ List<Pair<byte [], HColumnDescriptor>> columnModifications,
+ List<byte []> columnDeletions) throws IOException {
int waitInterval = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_WAIT_INTERVAL_MS,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_WAIT_INTERVAL_MS);
@@ -1645,7 +1663,45 @@ public class HMaster extends HasThread implements HMasterInterface,
int maxClosedRegions = conf.getInt(HConstants.MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE,
HConstants.DEFAULT_MASTER_SCHEMA_CHANGES_MAX_CONCURRENT_REGION_CLOSE);
- alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
+ return alterTable(tableName, columnAdditions, columnModifications, columnDeletions, waitInterval, maxClosedRegions);
+ }
+
+ /**
+ * This method rewrites all .regioninfo files for all regions belonging to the Table whose name is
+ * tableName
+ * @param tableName
+ * @return true if all .regioninfo files that belong to the table are successfully rewritten.
+ * @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
+ public boolean writeRegionInfo(byte[] tableName) throws Exception {
+ List<Pair<HRegionInfo, HServerAddress>> tableRegions = getTableRegions(tableName);
+ ExecutorService executor = Executors.newCachedThreadPool();
+ List<Future<Boolean>> futures = new ArrayList<>();
+ for (final Pair<HRegionInfo, HServerAddress> entry : tableRegions) {
+ Callable<Boolean> writer = new Callable<Boolean>() {
+
+ // return true if the write is successful
+ public Boolean call() {
+ HRegionInfo hri = entry.getFirst();
+ try {
+ return hri.writeToDisk(conf);
+ } catch (IOException e) {
+ LOG.error("Failed to write .regioninfo for Region: " + hri.getRegionNameAsString(), e);
+ return false;
+ }
+ }
+ };
+ Future<Boolean> future = executor.submit(writer);
+ futures.add(future);
+ }
+ for (Future<Boolean> f : futures) {
+ if (!f.get()) { // if any write was not successful
+ return false;
+ }
+ }
+ return true;
}
@Override
@@ -1654,7 +1710,7 @@ public class HMaster extends HasThread implements HMasterInterface,
Pair <Integer, Integer> p = new Pair<Integer, Integer>(0,0);
if (regionManager.getThrottledReopener(Bytes.toString(tableName)) != null) {
p = regionManager.getThrottledReopener(
- Bytes.toString(tableName)).getReopenStatus();
+ Bytes.toString(tableName)).getReopenStatus();
} else {
// Table is not reopening any regions return (0,0)
}
@@ -1663,21 +1719,21 @@ public class HMaster extends HasThread implements HMasterInterface,
@Override
public void addColumn(byte [] tableName, HColumnDescriptor column)
- throws IOException {
+ throws IOException {
alterTable(tableName, Arrays.asList(column), null, null);
}
@Override
public void modifyColumn(byte [] tableName, byte [] columnName,
- HColumnDescriptor descriptor)
- throws IOException {
+ HColumnDescriptor descriptor)
+ throws IOException {
alterTable(tableName, null, Arrays.asList(
new Pair<byte[], HColumnDescriptor>(columnName, descriptor)), null);
}
@Override
public void deleteColumn(final byte [] tableName, final byte [] c)
- throws IOException {
+ throws IOException {
alterTable(tableName, null, null,
Arrays.asList(KeyValue.parseColumn(c)[0]));
}
@@ -1718,27 +1774,27 @@ public class HMaster extends HasThread implements HMasterInterface,
*/
public List<Pair<HRegionInfo,HServerAddress>> getTableRegions(
final byte [] tableName)
- throws IOException {
+ throws IOException {
final ArrayList<Pair<HRegionInfo, HServerAddress>> result =
- Lists.newArrayList();
+ Lists.newArrayList();
if (!Bytes.equals(HConstants.META_TABLE_NAME, tableName)) {
MetaScannerVisitor visitor =
- new MetaScannerVisitor() {
- @Override
- public boolean processRow(Result data) throws IOException {
- if (data == null || data.size() <= 0)
- return true;
- Pair<HRegionInfo, HServerAddress> pair =
- metaRowToRegionPair(data);
- if (pair == null) return false;
- if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
- tableName)) {
- return false;
- }
- result.add(pair);
+ new MetaScannerVisitor() {
+ @Override
+ public boolean processRow(Result data) throws IOException {
+ if (data == null || data.size() <= 0)
return true;
+ Pair<HRegionInfo, HServerAddress> pair =
+ metaRowToRegionPair(data);
+ if (pair == null) return false;
+ if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
+ tableName)) {
+ return false;
}
+ result.add(pair);
+ return true;
+ }
};
MetaScanner.metaScan(conf, visitor, new StringBytes(tableName));
@@ -1748,7 +1804,7 @@ public class HMaster extends HasThread implements HMasterInterface,
for (MetaRegion mRegion: metaRegions) {
if (Bytes.equals(mRegion.getRegionInfo().getTableDesc().getName(), tableName)) {
result.add(new Pair<HRegionInfo, HServerAddress>
- (mRegion.getRegionInfo(), mRegion.getServer()));
+ (mRegion.getRegionInfo(), mRegion.getServer()));
}
}
}
@@ -1779,26 +1835,26 @@ public class HMaster extends HasThread implements HMasterInterface,
*/
Pair<HRegionInfo,HServerAddress> getTableRegionForRow(
final byte [] tableName, final byte [] rowKey)
- throws IOException {
+ throws IOException {
final AtomicReference<Pair<HRegionInfo, HServerAddress>> result =
- new AtomicReference<Pair<HRegionInfo, HServerAddress>>(null);
+ new AtomicReference<Pair<HRegionInfo, HServerAddress>>(null);
MetaScannerVisitor visitor =
- new MetaScannerVisitor() {
- @Override
- public boolean processRow(Result data) throws IOException {
- if (data == null || data.size() <= 0)
- return true;
- Pair<HRegionInfo, HServerAddress> pair =
- metaRowToRegionPair(data);
- if (pair == null) return false;
- if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
- tableName)) {
- return false;
- }
- result.set(pair);
+ new MetaScannerVisitor() {
+ @Override
+ public boolean processRow(Result data) throws IOException {
+ if (data == null || data.size() <= 0)
return true;
+ Pair<HRegionInfo, HServerAddress> pair =
+ metaRowToRegionPair(data);
+ if (pair == null) return false;
+ if (!Bytes.equals(pair.getFirst().getTableDesc().getName(),
+ tableName)) {
+ return false;
}
+ result.set(pair);
+ return true;
+ }
};
MetaScanner.metaScan(conf, visitor, new StringBytes(tableName), rowKey, 1);
@@ -1808,7 +1864,7 @@ public class HMaster extends HasThread implements HMasterInterface,
@SuppressWarnings("deprecation")
Pair<HRegionInfo,HServerAddress> getTableRegionFromName(
final byte [] regionName)
- throws IOException {
+ throws IOException {
byte [] tableName = HRegionInfo.parseRegionName(regionName)[0];
Set<MetaRegion> regions = regionManager.getMetaRegionsForTable(tableName);
@@ -1834,7 +1890,7 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
protected Result getFromMETA(final byte[] row, final byte[] family)
- throws IOException {
+ throws IOException {
MetaRegion meta = this.regionManager.getMetaRegionForRow(row);
HRegionInterface srvr = getMETAServer(meta);
Get get = new Get.Builder(row).addFamily(family).create();
@@ -1847,7 +1903,7 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
private HRegionInterface getMETAServer(final MetaRegion meta)
- throws IOException {
+ throws IOException {
return this.connection.getHRegionConnection(meta.getServer());
}
@@ -1858,18 +1914,18 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException if a remote or network exception occurs
*/
public HTableDescriptor getTableDescriptor(final byte [] tableName)
- throws IOException {
+ throws IOException {
return this.connection.getHTableDescriptor(new StringBytes(tableName));
}
@Override
public void modifyTable(final byte[] tableName, HConstants.Modify op,
Writable[] args)
- throws IOException {
+ throws IOException {
switch (op) {
case TABLE_SET_HTD:
if (args == null || args.length < 1 ||
- !(args[0] instanceof HTableDescriptor))
+ !(args[0] instanceof HTableDescriptor))
throw new IOException("SET_HTD request requires an HTableDescriptor");
HTableDescriptor htd = (HTableDescriptor) args[0];
LOG.info("modifyTable(SET_HTD): " + htd);
@@ -1883,7 +1939,7 @@ public class HMaster extends HasThread implements HMasterInterface,
if (args != null && args.length > 0) {
if (!(args[0] instanceof ImmutableBytesWritable))
throw new IOException(
- "request argument must be ImmutableBytesWritable");
+ "request argument must be ImmutableBytesWritable");
Pair<HRegionInfo,HServerAddress> pair = null;
if(tableName == null) {
byte [] regionName = ((ImmutableBytesWritable)args[0]).get();
@@ -1892,8 +1948,8 @@ public class HMaster extends HasThread implements HMasterInterface,
byte [] rowKey = ((ImmutableBytesWritable)args[0]).get();
pair = getTableRegionForRow(tableName, rowKey);
}
- LOG.info("About to " + op.toString() + " on "
- + Bytes.toStringBinary(tableName) + " and pair is " + pair);
+ LOG.info("About to " + op.toString() + " on "
+ + Bytes.toStringBinary(tableName) + " and pair is " + pair);
if (pair != null && pair.getSecond() != null) {
// If the column family name is specified, we need to perform a
// column family specific action instead of an action on the whole
@@ -1916,12 +1972,12 @@ public class HMaster extends HasThread implements HMasterInterface,
for (Pair<HRegionInfo,HServerAddress> pair: getTableRegions(tableName)) {
if (pair.getSecond() == null) continue; // undeployed
this.regionManager.startAction(pair.getFirst().getRegionName(),
- pair.getFirst(), pair.getSecond(), op);
+ pair.getFirst(), pair.getSecond(), op);
}
}
break;
- // format : {tableName row | region} splitPoint
+ // format : {tableName row | region} splitPoint
case TABLE_EXPLICIT_SPLIT:
if (args == null || args.length < (tableName == null? 2 : 1)) {
throw new IOException("incorrect number of arguments given");
@@ -1947,13 +2003,13 @@ public class HMaster extends HasThread implements HMasterInterface,
}
HRegionInfo r = pair.getFirst();
r.setSplitPoint(splitPoint);
- LOG.info("About to " + op.toString() + " on "
- + Bytes.toStringBinary(pair.getFirst().getTableDesc().getName())
- + " at " + Bytes.toStringBinary(splitPoint) + " and pair is "
- + pair);
+ LOG.info("About to " + op.toString() + " on "
+ + Bytes.toStringBinary(pair.getFirst().getTableDesc().getName())
+ + " at " + Bytes.toStringBinary(splitPoint) + " and pair is "
+ + pair);
if (pair.getSecond() != null) {
this.regionManager.startAction(pair.getFirst().getRegionName(),
- pair.getFirst(), pair.getSecond(), Modify.TABLE_SPLIT);
+ pair.getFirst(), pair.getSecond(), Modify.TABLE_SPLIT);
}
break;
@@ -1976,7 +2032,7 @@ public class HMaster extends HasThread implements HMasterInterface,
}
this.regionManager.getAssignmentManager().
- addTransientAssignment(serverAddress, hri);
+ addTransientAssignment(serverAddress, hri);
// Close the region so that it will be re-opened by the preferred host.
modifyTable(tableName, HConstants.Modify.CLOSE_REGION, new Writable[]{args[0]});
@@ -1986,7 +2042,7 @@ public class HMaster extends HasThread implements HMasterInterface,
case CLOSE_REGION:
if (args == null || args.length < 1 || args.length > 2) {
throw new IOException("Requires at least a region name; " +
- "or cannot have more than region name and servername");
+ "or cannot have more than region name and servername");
}
// Arguments are regionname and an optional server name.
byte [] regionname = ((ImmutableBytesWritable)args[0]).get();
@@ -2001,8 +2057,8 @@ public class HMaster extends HasThread implements HMasterInterface,
if (hostnameAndPort == null) {
// Get server from the .META. if it wasn't passed as argument
hostnameAndPort =
- Bytes.toString(rr.getValue(HConstants.CATALOG_FAMILY,
- HConstants.SERVER_QUALIFIER));
+ Bytes.toString(rr.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.SERVER_QUALIFIER));
}
// Take region out of the intransistions in case it got stuck there doing
// an open or whatever.
@@ -2010,11 +2066,11 @@ public class HMaster extends HasThread implements HMasterInterface,
// If hostnameAndPort is still null, then none, exit.
if (hostnameAndPort == null) break;
long startCode =
- Bytes.toLong(rr.getValue(HConstants.CATALOG_FAMILY,
- HConstants.STARTCODE_QUALIFIER));
+ Bytes.toLong(rr.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.STARTCODE_QUALIFIER));
String name = HServerInfo.getServerName(hostnameAndPort, startCode);
LOG.info("Marking " + hri.getRegionNameAsString() +
- " as closing on " + name + "; cleaning SERVER + STARTCODE; " +
+ " as closing on " + name + "; cleaning SERVER + STARTCODE; " +
"master will tell regionserver to close region on next heartbeat");
this.regionManager.setClosing(name, hri, hri.isOffline());
break;
@@ -2053,14 +2109,14 @@ public class HMaster extends HasThread implements HMasterInterface,
* @throws IOException
*/
static HRegionInfo getHRegionInfo(final byte[] row, final Result res)
- throws IOException {
+ throws IOException {
byte[] regioninfo = res.getValue(HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER);
if (regioninfo == null) {
StringBuilder sb = new StringBuilder();
NavigableMap<byte[], byte[]> infoMap =
- res.getFamilyMap(HConstants.CATALOG_FAMILY);
+ res.getFamilyMap(HConstants.CATALOG_FAMILY);
if (infoMap == null) {
return null;
}
@@ -2133,12 +2189,12 @@ public class HMaster extends HasThread implements HMasterInterface,
final Configuration conf) {
try {
Constructor<? extends HMaster> c =
- masterClass.getConstructor(Configuration.class);
+ masterClass.getConstructor(Configuration.class);
return c.newInstance(conf);
} catch (Exception e) {
throw new RuntimeException("Failed construction of " +
- "Master: " + masterClass.toString() +
- ((e.getCause() != null)? e.getCause().getMessage(): ""), e);
+ "Master: " + masterClass.toString() +
+ ((e.getCause() != null)? e.getCause().getMessage(): ""), e);
}
}
@@ -2210,14 +2266,14 @@ public class HMaster extends HasThread implements HMasterInterface,
RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
if (runtime != null) {
LOG.info("vmName=" + runtime.getVmName() + ", vmVendor=" +
- runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion());
+ runtime.getVmVendor() + ", vmVersion=" + runtime.getVmVersion());
LOG.info("vmInputArguments=" + runtime.getInputArguments());
}
// If 'local', defer to LocalHBaseCluster instance. Starts master
// and regionserver both in the one JVM.
if (LocalHBaseCluster.isLocal(conf)) {
final MiniZooKeeperCluster zooKeeperCluster =
- new MiniZooKeeperCluster();
+ new MiniZooKeeperCluster();
File zkDataPath = new File(conf.get("hbase.zookeeper.property.dataDir"));
int zkClientPort = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 0);
if (zkClientPort == 0) {
@@ -2234,11 +2290,11 @@ public class HMaster extends HasThread implements HMasterInterface,
throw new IOException(errorMsg);
}
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
- Integer.toString(clientPort));
+ Integer.toString(clientPort));
// Need to have the zk cluster shutdown when master is shutdown.
// Run a subclass that does the zk cluster shutdown on its way out.
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, 1, 1,
- LocalHMaster.class, HRegionServer.class);
+ LocalHMaster.class, HRegionServer.class);
((LocalHMaster)cluster.getMaster()).setZKCluster(zooKeeperCluster);
cluster.startup();
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 285db03..a68f1c3 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -59,7 +59,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -755,20 +754,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
// clash w/ a store/family name. There is possibility, but assumption is
// that its slim (don't want to use control character in filename because
//
- Path regioninfo = new Path(this.regiondir, REGIONINFO_FILE);
- if (this.fs.exists(regioninfo) &&
- this.fs.getFileStatus(regioninfo).getLen() > 0) {
- return;
- }
- FSDataOutputStream out = this.fs.create(regioninfo, true);
- try {
- this.regionInfo.write(out);
- out.write('\n');
- out.write('\n');
- out.write(Bytes.toBytes(this.regionInfo.toString()));
- } finally {
- out.close();
- }
+ this.regionInfo.writeToDisk(conf, fs);
}
public AtomicLong getMemstoreSize() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/75ed75d3/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
index 44738aa..1bbdd34 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsckFixRegionInfo.java
@@ -23,7 +23,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -45,7 +44,7 @@ import org.junit.experimental.categories.Category;
import java.io.IOException;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
/**
* Class to test the checkRegionInfo() and fixRegionInfo() methods of HBaseFsck.
@@ -80,17 +79,10 @@ public class TestHBaseFsckFixRegionInfo {
byte[] tableName = Bytes.toBytes("testFixRegionInfo");
- byte[][] splitKeys = {
- new byte[] { 1, 1, 1 },
- new byte[] { 2, 2, 2 },
- new byte[] { 3, 3, 3 },
- new byte[] { 4, 4, 4 },
- new byte[] { 5, 5, 5 },
- new byte[] { 6, 6, 6 },
- new byte[] { 7, 7, 7 },
- new byte[] { 8, 8, 8 },
- new byte[] { 9, 9, 9 },
- };
+ byte[][] splitKeys =
+ { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, new byte[] { 3, 3, 3 },
+ new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
+ new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
@@ -100,79 +92,77 @@ public class TestHBaseFsckFixRegionInfo {
HBaseFsck fsckToCorrupt = new HBaseFsck(TEST_UTIL.getConfiguration());
fsckToCorrupt.initAndScanRootMeta();
/*
- * The following line is necessary because some .regioninfo files (specifically the ROOT and the META)
- * become outdated immediately after initialization. This is because some parameters are immediately changed
- * after initialization, and these changes are not updated in the .regioninfo. So here we update all files.
- * This allows us to accurate measure the # of corrupted files later.
+ * The following line is necessary because some .regioninfo files (specifically the ROOT and the
+ * META) become outdated immediately after initialization. This is because some parameters are
+ * immediately changed after initialization, and these changes are not updated in the
+ * .regioninfo. So here we update all files. This allows us to accurate measure the # of
+ * corrupted files later.
*/
fsckToCorrupt.fixRegionInfo();
- //Randomly corrupt some files
+ // Randomly corrupt some files
int modifyCount = 0; // # of corrupted files
for (HbckInfo hbi : fsckToCorrupt.getRegionInfo().values()) {
if (Math.random() < 0.5) {
- Path tableDir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
- hbi.metaEntry.getTableDesc().getName());
+ Path tableDir =
+ HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), hbi.metaEntry.getTableDesc()
+ .getName());
Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
FileSystem fs = rootDir.getFileSystem(conf);
- Path regionPath = HRegion.getRegionDir(tableDir,
- hbi.metaEntry.getEncodedName());
+ Path regionPath = HRegion.getRegionDir(tableDir, hbi.metaEntry.getEncodedName());
Path regionInfoPath = new Path(regionPath, HRegion.REGIONINFO_FILE);
- //read the original .regioninfo into an HRegionInfo
+ // read the original .regioninfo into an HRegionInfo
FSDataInputStream in = fs.open(regionInfoPath);
- HRegionInfo hri_original = new HRegionInfo();
- hri_original.readFields(in);
+ HRegionInfo hriOriginal = new HRegionInfo();
+ hriOriginal.readFields(in);
in.close();
- //change the HRegionInfo
- HRegionInfo hri_modified = randomlyModifyRegion(hri_original);
+ // change the HRegionInfo
+ HRegionInfo hriModified = randomlyModifyRegion(hriOriginal);
- //rewrite the original .regioninfo
- FSDataOutputStream out = fs.create(regionInfoPath, true);
- hri_modified.write(out);
- out.write('\n');
- out.write('\n');
- out.write(Bytes.toBytes(hri_modified.toString()));
- out.close();
+ // rewrite the original .regioninfo
+ hriModified.writeToDisk(conf);
+ System.out.println("SUCCESSFULLY CORRUPTED:"
+ + !hriOriginal.toString().equals(hriModified.toString()));
+ System.out.println("Original " + hriOriginal.toString());
+ System.out.println("Modified: " + hriModified.toString());
modifyCount++;
}
}
- //we incorrectly rewrote some .regioninfo files, so some .regioninfos are incorrect
+ // we incorrectly rewrote some .regioninfo files, so some .regioninfos are incorrect
HBaseFsck fsckCorrupted = new HBaseFsck(TEST_UTIL.getConfiguration());
fsckCorrupted.initAndScanRootMeta();
Map<HRegionInfo, Path> risToRewrite = fsckCorrupted.checkRegionInfo();
- assertEquals(modifyCount, risToRewrite.size()); // # of files to rewrite must be the # of modified files
+ assertTrue("Expected " + modifyCount + " inconsistencies but saw " + risToRewrite.size()
+ + ".", risToRewrite.size() <= modifyCount); // # of files to rewrite is at most # of files modified
- //after fixing, we should see no errors
+ // after fixing, we should see no errors
HBaseFsck fsckFixed = new HBaseFsck(TEST_UTIL.getConfiguration());
fsckFixed.initAndScanRootMeta();
fsckFixed.fixRegionInfo();
risToRewrite = fsckFixed.checkRegionInfo();
- assertEquals(0, risToRewrite.size());
+ assertEquals("Expected 0 inconsistencies after fixing but saw " + risToRewrite.size() + ".", 0,
+ risToRewrite.size());
}
- HRegionInfo randomlyModifyRegion(HRegionInfo hri){
- HTableDescriptor tableDesc = hri.getTableDesc();
+ HRegionInfo randomlyModifyRegion(HRegionInfo hri) {
+ HRegionInfo hriModified = new HRegionInfo(hri);
+ HTableDescriptor tableDescOriginal = hriModified.getTableDesc();
+ HTableDescriptor tableDescModified = new HTableDescriptor(tableDescOriginal);
double rand = Math.random();
- if (rand < 0.2){
- tableDesc.setReadOnly(!tableDesc.isReadOnly());
+ if (rand < 0.25) {
+ tableDescModified.setReadOnly(!tableDescOriginal.isReadOnly());
+ } else if (rand < 0.5) {
+ tableDescModified.setDeferredLogFlush(!tableDescOriginal.isDeferredLogFlush());
+ } else if (rand < 0.75) {
+ tableDescModified.setWALDisabled(!tableDescOriginal.isWALDisabled());
+ } else {
+ tableDescModified.setMemStoreFlushSize((long) tableDescOriginal.getMemStoreFlushSize() + 1); // is incorrect
}
- else if (rand < 0.4){
- tableDesc.setDeferredLogFlush(!tableDesc.isDeferredLogFlush());
- }
- else if (rand < 0.6){
- tableDesc.setName(Bytes.toBytes("NEWNAME"));
- }
- else if (rand < 0.8){
- tableDesc.setMaxFileSize( (long) 100);
- }
- else{
- tableDesc.setMemStoreFlushSize( (long) 200);
- }
- hri.setTableDesc(tableDesc);
- return hri;
+ hriModified.setTableDesc(tableDescModified);
+ return hriModified;
}
}