You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2014/02/06 03:04:54 UTC
svn commit: r1565041 [1/3] - in /hbase/branches/hbase-10070:
hbase-client/src/main/java/org/apache/hadoop/hbase/
hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-client/src/tes...
Author: enis
Date: Thu Feb 6 02:04:53 2014
New Revision: 1565041
URL: http://svn.apache.org/r1565041
Log:
HBASE-10347 HRegionInfo changes for adding replicaId and MetaEditor/MetaReader changes for region replicas
Added:
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java
hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/TestRegionLocations.java
Modified:
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java
hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java
hbase/branches/hbase-10070/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
hbase/branches/hbase-10070/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/hbase-10070/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java
hbase/branches/hbase-10070/hbase-protocol/src/main/protobuf/HBase.proto
hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/client/CoprocessorHConnection.java
hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionInfo.java
Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Thu Feb 6 02:04:53 2014
@@ -29,7 +29,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import com.google.protobuf.HBaseZeroCopyByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -47,15 +46,34 @@ import org.apache.hadoop.hbase.util.Pair
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.io.DataInputBuffer;
+import com.google.protobuf.HBaseZeroCopyByteString;
import com.google.protobuf.InvalidProtocolBufferException;
/**
- * HRegion information.
- * Contains HRegion id, start and end keys, a reference to this HRegions' table descriptor, etc.
+ * Information about a region. A region is a range of keys in the whole keyspace of a table, an
+ * identifier (a timestamp) for differentiating between subset ranges (after region split)
+ * and a replicaId for differentiating the instance for the same range and some status information
+ * about the region.
+ *
+ * The region has a unique name which consists of the following fields:
+ * <li> tableName : The name of the table </li>
+ * <li> startKey : The startKey for the region. </li>
+ * <li> regionId : A timestamp when the region is created. </li>
+ * <li> replicaId : An id starting from 0 to differentiate replicas of the same region range
+ * but hosted in separated servers. The same region range can be hosted in multiple locations.</li>
+ * <li> encodedName : An MD5 encoded string for the region name.</li>
*
- * On a big cluster, each client will have thousands of instances of this object, often
- * 100 000 of them if not million. It's important to keep the object size as small
- * as possible.
+ * <br> Other than the fields in the region name, region info contains:
+ * <li> endKey : the endKey for the region (exclusive) </li>
+ * <li> split : Whether the region is split </li>
+ * <li> offline : Whether the region is offline </li>
+ *
+ * In 0.98 or before, a list of table's regions would fully cover the total keyspace, and at any
+ * point in time, a row key always belongs to a single region, which is hosted in a single server.
+ * In 0.99+, a region can have multiple instances (called replicas), and thus a range (or row) can
+ * correspond to multiple HRegionInfo's. These HRI's share the same fields however except the
+ * replicaId field. If the replicaId is not set, it defaults to 0, which is compatible with the
+ * previous behavior of a range corresponding to 1 region.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -118,6 +136,14 @@ public class HRegionInfo implements Comp
/** A non-capture group so that this can be embedded. */
public static final String ENCODED_REGION_NAME_REGEX = "(?:[a-f0-9]+)";
+ // to keep appended int's sorted in string format. Only allows 2 bytes to be
+ // sorted for replicaId
+ public static final String REPLICA_ID_FORMAT = "%04X";
+
+ public static final byte REPLICA_ID_DELIMITER = (byte)'_';
+
+ private static final int MAX_REPLICA_ID = 0xFFFF;
+ private static final int DEFAULT_REPLICA_ID = 0;
/**
* Does region name contain its encoded name?
* @param regionName region name
@@ -190,6 +216,7 @@ public class HRegionInfo implements Comp
public static final String NO_HASH = null;
private String encodedName = null;
private byte [] encodedNameAsBytes = null;
+ private int replicaId = DEFAULT_REPLICA_ID;
// Current TableName
private TableName tableName = null;
@@ -205,6 +232,7 @@ public class HRegionInfo implements Comp
result ^= Arrays.hashCode(this.endKey);
result ^= Boolean.valueOf(this.offLine).hashCode();
result ^= Arrays.hashCode(this.tableName.getName());
+ result ^= this.replicaId;
this.hashCode = result;
}
@@ -248,7 +276,6 @@ public class HRegionInfo implements Comp
this(tableName, startKey, endKey, false);
}
-
/**
* Construct HRegionInfo with explicit parameters
*
@@ -265,7 +292,6 @@ public class HRegionInfo implements Comp
this(tableName, startKey, endKey, split, System.currentTimeMillis());
}
-
/**
* Construct HRegionInfo with explicit parameters
*
@@ -280,7 +306,25 @@ public class HRegionInfo implements Comp
public HRegionInfo(final TableName tableName, final byte[] startKey,
final byte[] endKey, final boolean split, final long regionid)
throws IllegalArgumentException {
+ this(tableName, startKey, endKey, split, regionid, DEFAULT_REPLICA_ID);
+ }
+ /**
+ * Construct HRegionInfo with explicit parameters
+ *
+ * @param tableName the table descriptor
+ * @param startKey first key in region
+ * @param endKey end of key range
+ * @param split true if this region has split and we have daughter regions
+ * regions that may or may not hold references to this region.
+ * @param regionid Region id to use.
+ * @param replicaId the replicaId to use
+ * @throws IllegalArgumentException
+ */
+ public HRegionInfo(final TableName tableName, final byte[] startKey,
+ final byte[] endKey, final boolean split, final long regionid,
+ final int replicaId)
+ throws IllegalArgumentException {
super();
if (tableName == null) {
throw new IllegalArgumentException("TableName cannot be null");
@@ -288,8 +332,12 @@ public class HRegionInfo implements Comp
this.tableName = tableName;
this.offLine = false;
this.regionId = regionid;
+ this.replicaId = replicaId;
+ if (this.replicaId > MAX_REPLICA_ID) {
+ throw new IllegalArgumentException("ReplicaId cannot be greater than" + MAX_REPLICA_ID);
+ }
- this.regionName = createRegionName(this.tableName, startKey, regionId, true);
+ this.regionName = createRegionName(this.tableName, startKey, regionId, replicaId, true);
this.split = split;
this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone();
@@ -315,8 +363,14 @@ public class HRegionInfo implements Comp
this.hashCode = other.hashCode();
this.encodedName = other.getEncodedName();
this.tableName = other.tableName;
+ this.replicaId = other.replicaId;
}
+ public HRegionInfo(HRegionInfo other, int replicaId) {
+ this(other);
+ this.replicaId = replicaId;
+ this.setHashCode();
+ }
/**
* Make a region name of passed parameters.
@@ -350,6 +404,22 @@ public class HRegionInfo implements Comp
* Make a region name of passed parameters.
* @param tableName
* @param startKey Can be null
+ * @param regionid Region id (Usually timestamp from when region was created).
+ * @param replicaId
+ * @param newFormat should we create the region name in the new format
+ * (such that it contains its encoded name?).
+ * @return Region name made of passed tableName, startKey, id and replicaId
+ */
+ public static byte [] createRegionName(final TableName tableName,
+ final byte [] startKey, final long regionid, int replicaId, boolean newFormat) {
+ return createRegionName(tableName, startKey, Bytes.toBytes(Long.toString(regionid)),
+ replicaId, newFormat);
+ }
+
+ /**
+ * Make a region name of passed parameters.
+ * @param tableName
+ * @param startKey Can be null
* @param id Region id (Usually timestamp from when region was created).
* @param newFormat should we create the region name in the new format
* (such that it contains its encoded name?).
@@ -357,9 +427,35 @@ public class HRegionInfo implements Comp
*/
public static byte [] createRegionName(final TableName tableName,
final byte [] startKey, final byte [] id, boolean newFormat) {
- byte [] b = new byte [tableName.getName().length + 2 + id.length +
- (startKey == null? 0: startKey.length) +
- (newFormat ? (MD5_HEX_LENGTH + 2) : 0)];
+ return createRegionName(tableName, startKey, id, DEFAULT_REPLICA_ID, newFormat);
+ }
+ /**
+ * Make a region name of passed parameters.
+ * @param tableName
+ * @param startKey Can be null
+ * @param id Region id (Usually timestamp from when region was created).
+ * @param replicaId
+ * @param newFormat should we create the region name in the new format
+ * @return Region name made of passed tableName, startKey, id and replicaId
+ */
+ public static byte [] createRegionName(final TableName tableName,
+ final byte [] startKey, final byte [] id, final int replicaId, boolean newFormat) {
+ int len = tableName.getName().length + 2 + id.length +
+ (startKey == null? 0: startKey.length);
+ if (newFormat) {
+ len += MD5_HEX_LENGTH + 2;
+ }
+ byte[] replicaIdBytes = null;
+ // Special casing: replicaId is only appended if replicaId is greater than
+ // 0. This is because all regions in meta would have to be migrated to the new
+ // name otherwise
+ if (replicaId > 0) {
+ // use string representation for replica id
+ replicaIdBytes = Bytes.toBytes(String.format(REPLICA_ID_FORMAT, replicaId));
+ len += 1 + replicaIdBytes.length;
+ }
+
+ byte [] b = new byte [len];
int offset = tableName.getName().length;
System.arraycopy(tableName.getName(), 0, b, 0, offset);
@@ -372,11 +468,17 @@ public class HRegionInfo implements Comp
System.arraycopy(id, 0, b, offset, id.length);
offset += id.length;
+ if (replicaIdBytes != null) {
+ b[offset++] = REPLICA_ID_DELIMITER;
+ System.arraycopy(replicaIdBytes, 0, b, offset, replicaIdBytes.length);
+ offset += replicaIdBytes.length;
+ }
+
if (newFormat) {
//
// Encoded name should be built into the region name.
//
- // Use the region name thus far (namely, <tablename>,<startKey>,<id>)
+ // Use the region name thus far (namely, <tablename>,<startKey>,<id>_<replicaId>)
// to compute a MD5 hash to be used as the encoded name, and append
// it to the byte buffer.
//
@@ -447,6 +549,11 @@ public class HRegionInfo implements Comp
*/
public static byte [][] parseRegionName(final byte [] regionName)
throws IOException {
+ // Region name is of the format:
+ // tablename,startkey,regionIdTimestamp[_replicaId][.encodedName.]
+ // startkey can contain the delimiter (',') so we parse from the start and end
+
+ // parse from start
int offset = -1;
for (int i = 0; i < regionName.length; i++) {
if (regionName[i] == HConstants.DELIMITER) {
@@ -458,8 +565,27 @@ public class HRegionInfo implements Comp
byte[] tableName = new byte[offset];
System.arraycopy(regionName, 0, tableName, 0, offset);
offset = -1;
- for (int i = regionName.length - 1; i > 0; i--) {
- if(regionName[i] == HConstants.DELIMITER) {
+
+ int endOffset = regionName.length;
+ // check whether regionName contains encodedName
+ if (regionName.length > MD5_HEX_LENGTH + 2
+ && regionName[regionName.length-1] == ENC_SEPARATOR
+ && regionName[regionName.length-MD5_HEX_LENGTH-2] == ENC_SEPARATOR) {
+ endOffset = endOffset - MD5_HEX_LENGTH - 2;
+ }
+
+ // parse from end
+ byte[] replicaId = null;
+ int idEndOffset = endOffset;
+ for (int i = endOffset - 1; i > 0; i--) {
+ if (regionName[i] == REPLICA_ID_DELIMITER) { //replicaId may or may not be present
+ replicaId = new byte[endOffset - i - 1];
+ System.arraycopy(regionName, i + 1, replicaId, 0,
+ endOffset - i - 1);
+ idEndOffset = i;
+ // do not break, continue to search for id
+ }
+ if (regionName[i] == HConstants.DELIMITER) {
offset = i;
break;
}
@@ -471,13 +597,17 @@ public class HRegionInfo implements Comp
System.arraycopy(regionName, tableName.length + 1, startKey, 0,
offset - tableName.length - 1);
}
- byte [] id = new byte[regionName.length - offset - 1];
+ byte [] id = new byte[idEndOffset - offset - 1];
System.arraycopy(regionName, offset + 1, id, 0,
- regionName.length - offset - 1);
- byte [][] elements = new byte[3][];
+ idEndOffset - offset - 1);
+ byte [][] elements = new byte[replicaId == null ? 3 : 4][];
elements[0] = tableName;
elements[1] = startKey;
elements[2] = id;
+ if (replicaId != null) {
+ elements[3] = replicaId;
+ }
+
return elements;
}
@@ -631,7 +761,6 @@ public class HRegionInfo implements Comp
this.offLine = offLine;
}
-
/**
* @return True if this is a split parent region.
*/
@@ -644,6 +773,14 @@ public class HRegionInfo implements Comp
}
/**
+ * Returns the region replica id
+ * @return returns region replica id
+ */
+ public int getReplicaId() {
+ return replicaId;
+ }
+
+ /**
* @see java.lang.Object#toString()
*/
@Override
@@ -654,7 +791,8 @@ public class HRegionInfo implements Comp
Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" +
Bytes.toStringBinary(this.endKey) + "'" +
(isOffline()? ", OFFLINE => true": "") +
- (isSplit()? ", SPLIT => true": "") + "}";
+ (isSplit()? ", SPLIT => true": "") +
+ ((replicaId > 0)? ", REPLICA_ID => " + replicaId : "") + "}";
}
/**
@@ -766,6 +904,7 @@ public class HRegionInfo implements Comp
// Comparable
//
+ @Override
public int compareTo(HRegionInfo o) {
if (o == null) {
return 1;
@@ -806,6 +945,9 @@ public class HRegionInfo implements Comp
return -1;
}
+ int replicaDiff = this.getReplicaId() - o.getReplicaId();
+ if (replicaDiff != 0) return replicaDiff;
+
if (this.offLine == o.offLine)
return 0;
if (this.offLine == true) return -1;
@@ -849,6 +991,7 @@ public class HRegionInfo implements Comp
}
builder.setOffline(info.isOffline());
builder.setSplit(info.isSplit());
+ builder.setReplicaId(info.getReplicaId());
return builder.build();
}
@@ -866,6 +1009,7 @@ public class HRegionInfo implements Comp
return FIRST_META_REGIONINFO;
}
long regionId = proto.getRegionId();
+ int replicaId = proto.hasReplicaId() ? proto.getReplicaId() : DEFAULT_REPLICA_ID;
byte[] startKey = null;
byte[] endKey = null;
if (proto.hasStartKey()) {
@@ -881,7 +1025,7 @@ public class HRegionInfo implements Comp
HRegionInfo hri = new HRegionInfo(
tableName,
startKey,
- endKey, split, regionId);
+ endKey, split, regionId, replicaId);
if (proto.hasOffline()) {
hri.setOffline(proto.getOffline());
}
@@ -980,7 +1124,9 @@ public class HRegionInfo implements Comp
* @return A pair of the {@link HRegionInfo} and the {@link ServerName}
* (or null for server address if no address set in hbase:meta).
* @throws IOException
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static Pair<HRegionInfo, ServerName> getHRegionInfoAndServerName(final Result r) {
HRegionInfo info =
getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
@@ -994,7 +1140,9 @@ public class HRegionInfo implements Comp
* table Result.
* @param data a Result object from the catalog table scan
* @return HRegionInfo or null
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static HRegionInfo getHRegionInfo(Result data) {
return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
}
@@ -1005,21 +1153,25 @@ public class HRegionInfo implements Comp
* @param data a Result object from the catalog table scan
* @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
* parent
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
return new PairOfSameType<HRegionInfo>(splitA, splitB);
}
-
+
/**
* Returns the merge regions by reading the corresponding columns of the catalog table
* Result.
* @param data a Result object from the catalog table scan
* @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
* parent
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) throws IOException {
HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
@@ -1035,7 +1187,9 @@ public class HRegionInfo implements Comp
* {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or
* {@link HConstants#REGIONINFO_QUALIFIER}.
* @return An HRegionInfo instance or null.
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
Cell cell = r.getColumnLatestCell(
HConstants.CATALOG_FAMILY, qualifier);
@@ -1044,10 +1198,9 @@ public class HRegionInfo implements Comp
}
/**
- * Returns a {@link ServerName} from catalog table {@link Result}.
- * @param r Result to pull from
- * @return A ServerName instance or null if necessary fields not found or empty.
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static ServerName getServerName(final Result r) {
Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
if (cell == null || cell.getValueLength() == 0) return null;
@@ -1065,7 +1218,9 @@ public class HRegionInfo implements Comp
* E.g. the seqNum when the result of {@link #getServerName(Result)} was written.
* @param r Result to pull the seqNum from
* @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
+ * @deprecated use MetaReader methods for interacting with meta layouts
*/
+ @Deprecated
public static long getSeqNumDuringOpen(final Result r) {
Cell cell = r.getColumnLatestCell(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER);
if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
@@ -1185,5 +1340,4 @@ public class HRegionInfo implements Comp
}
return false;
}
-
}
Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java Thu Feb 6 02:04:53 2014
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hbase.util.Addressing;
/**
@@ -32,9 +31,10 @@ import org.apache.hadoop.hbase.util.Addr
* On a big cluster, each client will have thousands of instances of this object, often
* 100 000 of them if not million. It's important to keep the object size as small
* as possible.
+ * <br>This interface has been marked InterfaceAudience.Public in 0.96 and 0.98, it is
+ * no longer the case.
*/
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@InterfaceAudience.Private
public class HRegionLocation implements Comparable<HRegionLocation> {
private final HRegionInfo regionInfo;
private final ServerName serverName;
@@ -112,6 +112,7 @@ public class HRegionLocation implements
return serverName;
}
+ @Override
public int compareTo(HRegionLocation o) {
return serverName.compareTo(o.getServerName());
}
Added: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java?rev=1565041&view=auto
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java (added)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLocations.java Thu Feb 6 02:04:53 2014
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import java.util.Collection;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Container for holding a list of {@link HRegionLocation}'s that correspond to the
+ * same range. The list is indexed by the replicaId. This is an immutable list,
+ * however mutation operations are provided which returns a new List via copy-on-write
+ * (assuming small number of locations)
+ */
+@InterfaceAudience.Private
+public class RegionLocations {
+
+ private final int numNonNullElements;
+ private final HRegionLocation[] locations; // replicaId -> HRegionLocation.
+
+ /**
+ * Constructs the region location list. The locations array should
+ * contain all the locations for known replicas for the region, and should be
+ * sorted in replicaId ascending order.
+ * @param locations an array of HRegionLocations for the same region range
+ */
+ public RegionLocations(HRegionLocation... locations) {
+ int numNonNullElements = 0;
+ int maxReplicaId = -1;
+ for (HRegionLocation loc : locations) {
+ if (loc != null) {
+ numNonNullElements++;
+ if (loc.getRegionInfo().getReplicaId() > maxReplicaId) {
+ maxReplicaId = loc.getRegionInfo().getReplicaId();
+ }
+ }
+ }
+ this.numNonNullElements = numNonNullElements;
+
+ if (maxReplicaId + 1 == locations.length) {
+ this.locations = locations;
+ } else {
+ this.locations = new HRegionLocation[maxReplicaId + 1];
+ for (HRegionLocation loc : locations) {
+ if (loc != null) {
+ this.locations[loc.getRegionInfo().getReplicaId()] = loc;
+ }
+ }
+ }
+ }
+
+ public RegionLocations(Collection<HRegionLocation> locations) {
+ this(locations.toArray(new HRegionLocation[locations.size()]));
+ }
+
+ /**
+ * Returns the size of the list even if some of the elements
+ * might be null.
+ * @return the size of the list (corresponding to the max replicaId)
+ */
+ public int size() {
+ return locations.length;
+ }
+
+ /**
+ * Returns the size of not-null locations
+ * @return the size of not-null locations
+ */
+ public int numNonNullElements() {
+ return numNonNullElements;
+ }
+
+ /**
+ * Returns whether there are non-null elements in the list
+ * @return whether there are non-null elements in the list
+ */
+ public boolean isEmpty() {
+ return numNonNullElements == 0;
+ }
+
+ /**
+ * Returns a new HRegionLocationList with the locations removed (set to null)
+ * which have the destination server as given.
+ * @param serverName the serverName to remove locations of
+ * @return an HRegionLocationList object with removed locations or the same object
+ * if nothing is removed
+ */
+ public RegionLocations removeByServer(ServerName serverName) {
+ HRegionLocation[] newLocations = null;
+ for (int i = 0; i < locations.length; i++) {
+ // check whether something to remove
+ if (locations[i] != null && serverName.equals(locations[i].getServerName())) {
+ if (newLocations == null) { //first time
+ newLocations = new HRegionLocation[locations.length];
+ System.arraycopy(locations, 0, newLocations, 0, i);
+ }
+ newLocations[i] = null;
+ } else if (newLocations != null) {
+ newLocations[i] = locations[i];
+ }
+ }
+ return newLocations == null ? this : new RegionLocations(newLocations);
+ }
+
+ /**
+ * Removes the given location from the list
+ * @param location the location to remove
+ * @return an HRegionLocationList object with removed locations or the same object
+ * if nothing is removed
+ */
+ public RegionLocations remove(HRegionLocation location) {
+ HRegionLocation[] newLocations = null;
+ for (int i = 0; i < locations.length; i++) {
+ // check whether something to remove. HRL.compareTo() compares ONLY the
+ // serverName. We want to compare the HRI's as well.
+ if (locations[i] != null
+ && location.getRegionInfo().equals(locations[i].getRegionInfo())
+ && location.equals(locations[i])) {
+ if (newLocations == null) { //first time
+ newLocations = new HRegionLocation[locations.length];
+ System.arraycopy(locations, 0, newLocations, 0, i);
+ }
+ newLocations[i] = null;
+ } else if (newLocations != null) {
+ newLocations[i] = locations[i];
+ }
+ }
+ return newLocations == null ? this : new RegionLocations(newLocations);
+ }
+
+ /**
+ * Merges this HRegionLocation list with the given list assuming
+ * same range, and keeping the most up to date version of the
+ * HRegionLocation entries from either list according to seqNum. If seqNums
+ * are equal, the location from the argument (other) is taken.
+ * @param other the locations to merge with
+ * @return an HRegionLocationList object with merged locations or the same object
+ * if nothing is merged
+ */
+ public RegionLocations mergeLocations(RegionLocations other) {
+ assert other != null;
+
+ HRegionLocation[] newLocations = null;
+
+ int max = Math.max(this.locations.length, other.locations.length);
+
+ for (int i = 0; i < max; i++) {
+ HRegionLocation thisLoc = this.getRegionLocation(i);
+ HRegionLocation otherLoc = other.getRegionLocation(i);
+
+ HRegionLocation selectedLoc = selectRegionLocation(thisLoc,
+ otherLoc, true, false);
+
+ if (selectedLoc != thisLoc) {
+ if (newLocations == null) {
+ newLocations = new HRegionLocation[max];
+ System.arraycopy(locations, 0, newLocations, 0, i);
+ }
+ }
+ if (newLocations != null) {
+ newLocations[i] = selectedLoc;
+ }
+ }
+
+ return newLocations == null ? this : new RegionLocations(newLocations);
+ }
+
+ private HRegionLocation selectRegionLocation(HRegionLocation oldLocation,
+ HRegionLocation location, boolean checkForEquals, boolean force) {
+ if (location == null) {
+ return oldLocation == null ? null : oldLocation;
+ }
+
+ if (oldLocation == null) {
+ return location;
+ }
+
+ if (force
+ || isGreaterThan(location.getSeqNum(), oldLocation.getSeqNum(), checkForEquals)) {
+ return location;
+ }
+ return oldLocation;
+ }
+
+ /**
+ * Updates the location with new only if the new location has a higher
+ * seqNum than the old one or force is true.
+ * @param location the location to add or update
+ * @param checkForEquals whether to update the location if seqNums for the
+ * HRegionLocations for the old and new location are the same
+ * @param force whether to force update
+ * @return an HRegionLocationList object with updated locations or the same object
+ * if nothing is updated
+ */
+ public RegionLocations updateLocation(HRegionLocation location,
+ boolean checkForEquals, boolean force) {
+ assert location != null;
+
+ int replicaId = location.getRegionInfo().getReplicaId();
+
+ HRegionLocation oldLoc = getRegionLocation(location.getRegionInfo().getReplicaId());
+ HRegionLocation selectedLoc = selectRegionLocation(oldLoc, location,
+ checkForEquals, force);
+
+ if (selectedLoc == oldLoc) {
+ return this;
+ }
+ HRegionLocation[] newLocations = new HRegionLocation[Math.max(locations.length, replicaId +1)];
+ System.arraycopy(locations, 0, newLocations, 0, locations.length);
+ newLocations[replicaId] = location;
+ return new RegionLocations(newLocations);
+ }
+
+ private boolean isGreaterThan(long a, long b, boolean checkForEquals) {
+ return a > b || (checkForEquals && (a == b));
+ }
+
+ public HRegionLocation getRegionLocation(int replicaId) {
+ if (replicaId >= locations.length) {
+ return null;
+ }
+ return locations[replicaId];
+ }
+
+ /**
+ * Returns the region location from the list for matching regionName, which can
+ * be regionName or encodedRegionName
+ * @param regionName regionName or encodedRegionName
+ * @return HRegionLocation found or null
+ */
+ public HRegionLocation getRegionLocationByRegionName(byte[] regionName) {
+ for (HRegionLocation loc : locations) {
+ if (loc != null) {
+ if (Bytes.equals(loc.getRegionInfo().getRegionName(), regionName)
+ || Bytes.equals(loc.getRegionInfo().getEncodedNameAsBytes(), regionName)) {
+ return loc;
+ }
+ }
+ }
+ return null;
+ }
+
+ public HRegionLocation[] getRegionLocations() {
+ return locations;
+ }
+
+ /**
+ * Returns the first not-null region location in the list
+ */
+ public HRegionLocation getRegionLocation() {
+ for (HRegionLocation loc : locations) {
+ if (loc != null) {
+ return loc;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("[");
+ for (HRegionLocation loc : locations) {
+ if (loc != null) {
+ if (builder.length() > 1) {
+ builder.append(", ");
+ }
+ builder.append(loc);
+ }
+ }
+ builder.append("]");
+ return builder.toString();
+ }
+
+}
Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Thu Feb 6 02:04:53 2014
@@ -17,37 +17,83 @@
*/
package org.apache.hadoop.hbase.catalog;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.PairOfSameType;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
+import com.google.common.annotations.VisibleForTesting;
/**
* Reads region and assignment information from <code>hbase:meta</code>.
*/
@InterfaceAudience.Private
public class MetaReader {
+
+ /*
+ * HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the
+ * same table range (table, startKey, endKey). For every range, there will be at least one
+ * HRI defined which is called default replica.
+ *
+ * Meta layout (as of 0.98 + HBASE-10070) is like:
+ * For each table range, there is a single row, formatted like:
+ * <tableName>,<startKey>,<regionId>,<encodedRegionName>. This row corresponds to the regionName
+ * of the default region replica.
+ * Columns are:
+ * info:regioninfo => contains serialized HRI for the default region replica
+ * info:server => contains hostname:port (in string form) for the server hosting
+ * the default regionInfo replica
+ * info:server_<replicaId> => contains hostname:port (in string form) for the server hosting the
+ * regionInfo replica with replicaId
+ * info:serverstartcode => contains server start code (in binary long form) for the server
+ * hosting the default regionInfo replica
+ * info:serverstartcode_<replicaId> => contains server start code (in binary long form) for the
+ * server hosting the regionInfo replica with replicaId
+ * info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time
+ * the server opened the region with default replicaId
+ * info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region at
+ * the time the server opened the region with replicaId
+ * info:splitA => contains a serialized HRI for the first daughter region if the
+ * region is split
+ * info:splitB => contains a serialized HRI for the second daughter region if the
+ * region is split
+ * info:mergeA => contains a serialized HRI for the first parent region if the
+ * region is the result of a merge
+ * info:mergeB => contains a serialized HRI for the second parent region if the
+ * region is the result of a merge
+ *
+ * The actual layout of meta should be encapsulated inside MetaReader and MetaEditor methods,
+ * and should not leak out of those (through Result objects, etc)
+ */
+
// TODO: Strip CatalogTracker from this class. Its all over and in the end
// its only used to get its Configuration so we can get associated
// Connection.
@@ -63,59 +109,12 @@ public class MetaReader {
META_REGION_PREFIX, 0, len);
}
- /**
- * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
- * tables in the specified set of disabled tables.
- * @param catalogTracker
- * @param disabledTables set of disabled tables that will not be returned
- * @return Returns a map of every region to it's currently assigned server,
- * according to META. If the region does not have an assignment it will have
- * a null value in the map.
- * @throws IOException
- */
- public static Map<HRegionInfo, ServerName> fullScan(
- CatalogTracker catalogTracker, final Set<TableName> disabledTables)
- throws IOException {
- return fullScan(catalogTracker, disabledTables, false);
- }
+ /** The delimiter for meta columns for replicaIds > 0 */
+ protected static final char META_REPLICA_ID_DELIMITER = '_';
- /**
- * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
- * tables in the specified set of disabled tables.
- * @param catalogTracker
- * @param disabledTables set of disabled tables that will not be returned
- * @param excludeOfflinedSplitParents If true, do not include offlined split
- * parents in the return.
- * @return Returns a map of every region to it's currently assigned server,
- * according to META. If the region does not have an assignment it will have
- * a null value in the map.
- * @throws IOException
- */
- public static Map<HRegionInfo, ServerName> fullScan(
- CatalogTracker catalogTracker, final Set<TableName> disabledTables,
- final boolean excludeOfflinedSplitParents)
- throws IOException {
- final Map<HRegionInfo, ServerName> regions =
- new TreeMap<HRegionInfo, ServerName>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
- HRegionInfo hri = region.getFirst();
- if (hri == null) return true;
- if (hri.getTable() == null) return true;
- if (disabledTables.contains(
- hri.getTable())) return true;
- // Are we to include split parents in the list?
- if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
- regions.put(hri, region.getSecond());
- return true;
- }
- };
- fullScan(catalogTracker, v);
- return regions;
- }
+ /** A regex for parsing server columns from meta. See above javadoc for meta layout */
+ private static final Pattern SERVER_COLUMN_PATTERN
+ = Pattern.compile("^server(_[0-9a-fA-F]{4})?$");
/**
* Performs a full scan of <code>hbase:meta</code>.
@@ -206,33 +205,81 @@ public class MetaReader {
}
/**
- * Reads the location of the specified region
+ * Gets the region info and assignment for the specified region.
* @param catalogTracker
- * @param regionName region whose location we are after
- * @return location of region as a {@link ServerName} or null if not found
+ * @param regionName Region to lookup.
+ * @return Location and HRegionInfo for <code>regionName</code>
* @throws IOException
+ * @deprecated use {@link #getRegionLocation(CatalogTracker, byte[])} instead
*/
- static ServerName readRegionLocation(CatalogTracker catalogTracker,
- byte [] regionName)
+ @Deprecated
+ public static Pair<HRegionInfo, ServerName> getRegion(
+ CatalogTracker catalogTracker, byte [] regionName)
throws IOException {
- Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
- return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
+ HRegionLocation location = getRegionLocation(catalogTracker, regionName);
+ return location == null
+ ? null
+ : new Pair<HRegionInfo, ServerName>(location.getRegionInfo(), location.getServerName());
}
/**
- * Gets the region info and assignment for the specified region.
+ * Returns the HRegionLocation from meta for the given region
* @param catalogTracker
- * @param regionName Region to lookup.
- * @return Location and HRegionInfo for <code>regionName</code>
+ * @param regionName
+ * @return HRegionLocation for the given region
* @throws IOException
*/
- public static Pair<HRegionInfo, ServerName> getRegion(
- CatalogTracker catalogTracker, byte [] regionName)
- throws IOException {
- Get get = new Get(regionName);
+ public static HRegionLocation getRegionLocation(CatalogTracker catalogTracker,
+ byte[] regionName) throws IOException {
+ byte[] row = regionName;
+ HRegionInfo parsedInfo = null;
+ try {
+ parsedInfo = parseRegionInfoFromRegionName(regionName);
+ row = getMetaKeyForRegion(parsedInfo);
+ } catch (Exception parseEx) {
+ LOG.warn("Received parse exception:" + parseEx);
+ }
+ Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
Result r = get(getCatalogHTable(catalogTracker), get);
- return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
+ RegionLocations locations = getRegionLocations(r);
+ return locations == null
+ ? null
+ : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
+ }
+
+ /**
+ * Returns the HRegionLocation from meta for the given region
+ * @param catalogTracker
+ * @param regionInfo
+ * @return HRegionLocation for the given region
+ * @throws IOException
+ */
+ public static HRegionLocation getRegionLocation(CatalogTracker catalogTracker,
+ HRegionInfo regionInfo) throws IOException {
+ byte[] row = getMetaKeyForRegion(regionInfo);
+ Get get = new Get(row);
+ get.addFamily(HConstants.CATALOG_FAMILY);
+ Result r = get(getCatalogHTable(catalogTracker), get);
+ return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+ }
+
+ /** Returns the row key to use for this regionInfo */
+ protected static byte[] getMetaKeyForRegion(HRegionInfo regionInfo) {
+ return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName();
+ }
+
+ /** Returns an HRI parsed from this regionName. Not all the fields of the HRI
+ * is stored in the name, so the returned object should only be used for the fields
+ * in the regionName.
+ */
+ protected static HRegionInfo parseRegionInfoFromRegionName(byte[] regionName)
+ throws IOException {
+ byte[][] fields = HRegionInfo.parseRegionName(regionName);
+ long regionId = Long.parseLong(Bytes.toString(fields[2]));
+ int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0;
+ return new HRegionInfo(
+ TableName.valueOf(fields[0]), fields[1], fields[1], false, regionId, replicaId);
}
/**
@@ -257,10 +304,8 @@ public class MetaReader {
public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
CatalogTracker catalogTracker, byte[] regionName) throws IOException {
Result result = getRegionResult(catalogTracker, regionName);
- HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
- HConstants.MERGEA_QUALIFIER);
- HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
- HConstants.MERGEB_QUALIFIER);
+ HRegionInfo mergeA = getHRegionInfo(result, HConstants.MERGEA_QUALIFIER);
+ HRegionInfo mergeB = getHRegionInfo(result, HConstants.MERGEB_QUALIFIER);
if (mergeA == null && mergeB == null) {
return null;
}
@@ -288,8 +333,12 @@ public class MetaReader {
@Override
public boolean visit(Result r) throws IOException {
- this.current =
- HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
+ RegionLocations locations = getRegionLocations(r);
+ if (locations == null || locations.getRegionLocation().getRegionInfo() == null) {
+ LOG.warn("No serialized HRegionInfo in " + r);
+ return true;
+ }
+ this.current = locations.getRegionLocation().getRegionInfo();
if (this.current == null) {
LOG.warn("No serialized HRegionInfo in " + r);
return true;
@@ -437,28 +486,33 @@ public class MetaReader {
// Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
- private Pair<HRegionInfo, ServerName> current = null;
+ private RegionLocations current = null;
@Override
public boolean visit(Result r) throws IOException {
- HRegionInfo hri =
- HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
- if (hri == null) {
+ current = getRegionLocations(r);
+ if (current == null || current.getRegionLocation().getRegionInfo() == null) {
LOG.warn("No serialized HRegionInfo in " + r);
return true;
}
+ HRegionInfo hri = current.getRegionLocation().getRegionInfo();
if (!isInsideTable(hri, tableName)) return false;
if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
- ServerName sn = HRegionInfo.getServerName(r);
- // Populate this.current so available when we call #add
- this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
// Else call super and add this Result to the collection.
return super.visit(r);
}
@Override
void add(Result r) {
- this.results.add(this.current);
+ if (current == null) {
+ return;
+ }
+ for (HRegionLocation loc : current.getRegionLocations()) {
+ if (loc != null) {
+ this.results.add(new Pair<HRegionInfo, ServerName>(
+ loc.getRegionInfo(), loc.getServerName()));
+ }
+ }
}
};
fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
@@ -482,19 +536,18 @@ public class MetaReader {
@Override
void add(Result r) {
if (r == null || r.isEmpty()) return;
- ServerName sn = HRegionInfo.getServerName(r);
- if (sn != null && sn.equals(serverName)) this.results.add(r);
+ RegionLocations locations = getRegionLocations(r);
+ if (locations == null) return;
+ for (HRegionLocation loc : locations.getRegionLocations()) {
+ if (loc != null) {
+ if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
+ hris.put(loc.getRegionInfo(), r);
+ }
+ }
+ }
}
};
fullScan(catalogTracker, v);
- List<Result> results = v.getResults();
- if (results != null && !results.isEmpty()) {
- // Convert results to Map keyed by HRI
- for (Result r: results) {
- Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
- if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
- }
- }
return hris;
}
@@ -505,8 +558,13 @@ public class MetaReader {
public boolean visit(Result r) throws IOException {
if (r == null || r.isEmpty()) return true;
LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
- HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
- LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
+ RegionLocations locations = getRegionLocations(r);
+ if (locations == null) return true;
+ for (HRegionLocation loc : locations.getRegionLocations()) {
+ if (loc != null) {
+ LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo());
+ }
+ }
return true;
}
};
@@ -551,6 +609,215 @@ public class MetaReader {
}
/**
+ * Returns the column family used for meta columns.
+ * @return HConstants.CATALOG_FAMILY.
+ */
+ protected static byte[] getFamily() {
+ return HConstants.CATALOG_FAMILY;
+ }
+
+ /**
+ * Returns the column qualifier for serialized region info
+ * @return HConstants.REGIONINFO_QUALIFIER
+ */
+ protected static byte[] getRegionInfoColumn() {
+ return HConstants.REGIONINFO_QUALIFIER;
+ }
+
+ /**
+ * Returns the column qualifier for server column for replicaId
+ * @param replicaId the replicaId of the region
+ * @return a byte[] for server column qualifier
+ */
+ protected static byte[] getServerColumn(int replicaId) {
+ return replicaId == 0
+ ? HConstants.SERVER_QUALIFIER
+ : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+ + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
+ }
+
+ /**
+ * Returns the column qualifier for server start code column for replicaId
+ * @param replicaId the replicaId of the region
+ * @return a byte[] for server start code column qualifier
+ */
+ protected static byte[] getStartCodeColumn(int replicaId) {
+ return replicaId == 0
+ ? HConstants.STARTCODE_QUALIFIER
+ : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+ + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
+ }
+
+ /**
+ * Returns the column qualifier for seqNum column for replicaId
+ * @param replicaId the replicaId of the region
+ * @return a byte[] for seqNum column qualifier
+ */
+ protected static byte[] getSeqNumColumn(int replicaId) {
+ return replicaId == 0
+ ? HConstants.SEQNUM_QUALIFIER
+ : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+ + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
+ }
+
+ /**
+ * Parses the replicaId from the server column qualifier. See top of the class javadoc
+ * for the actual meta layout
+ * @param serverColumn the column qualifier
+ * @return an int for the replicaId
+ */
+ @VisibleForTesting
+ static int parseReplicaIdFromServerColumn(byte[] serverColumn) {
+ String serverStr = Bytes.toString(serverColumn);
+
+ Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr);
+ if (matcher.matches() && matcher.groupCount() > 0) {
+ String group = matcher.group(1);
+ if (group != null && group.length() > 0) {
+ return Integer.parseInt(group.substring(1), 16);
+ } else {
+ return 0;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Returns a {@link ServerName} from catalog table {@link Result}.
+ * @param r Result to pull from
+ * @return A ServerName instance or null if necessary fields not found or empty.
+ */
+ private static ServerName getServerName(final Result r, final int replicaId) {
+ byte[] serverColumn = getServerColumn(replicaId);
+ Cell cell = r.getColumnLatestCell(getFamily(), serverColumn);
+ if (cell == null || cell.getValueLength() == 0) return null;
+ String hostAndPort = Bytes.toString(
+ cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ byte[] startcodeColumn = getStartCodeColumn(replicaId);
+ cell = r.getColumnLatestCell(getFamily(), startcodeColumn);
+ if (cell == null || cell.getValueLength() == 0) return null;
+ return ServerName.valueOf(hostAndPort,
+ Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
+ }
+
+ /**
+ * The latest seqnum that the server writing to meta observed when opening the region.
+ * E.g. the seqNum when the result of {@link #getServerName(Result)} was written.
+ * @param r Result to pull the seqNum from
+ * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
+ */
+ private static long getSeqNumDuringOpen(final Result r, final int replicaId) {
+ Cell cell = r.getColumnLatestCell(getFamily(), getSeqNumColumn(replicaId));
+ if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
+ return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ }
+
+ /**
+ * Returns an HRegionLocationList extracted from the result.
+ * @return an HRegionLocationList containing all locations for the region range
+ */
+ public static RegionLocations getRegionLocations(final Result r) {
+ if (r == null) return null;
+ HRegionInfo regionInfo = getHRegionInfo(r, getRegionInfoColumn());
+ if (regionInfo == null) return null;
+
+ List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1);
+ NavigableMap<byte[],NavigableMap<byte[],byte[]>> familyMap = r.getNoVersionMap();
+
+ locations.add(getRegionLocation(r, regionInfo, 0));
+
+ NavigableMap<byte[], byte[]> infoMap = familyMap.get(getFamily());
+ if (infoMap == null) return new RegionLocations(locations);
+
+ // iterate until all serverName columns are seen
+ int replicaId = 0;
+ byte[] serverColumn = getServerColumn(replicaId);
+ SortedMap<byte[], byte[]> serverMap = infoMap.tailMap(serverColumn, false);
+ if (serverMap.isEmpty()) return new RegionLocations(locations);
+
+ for (Entry<byte[], byte[]> entry : serverMap.entrySet()) {
+ replicaId = parseReplicaIdFromServerColumn(entry.getKey());
+ if (replicaId < 0) {
+ break;
+ }
+
+ locations.add(getRegionLocation(r, regionInfo, replicaId));
+ }
+
+ return new RegionLocations(locations);
+ }
+
+ /**
+ * Returns the HRegionLocation parsed from the given meta row Result
+ * for the given regionInfo and replicaId. The regionInfo can be the default region info
+ * for the replica.
+ * @param r the meta row result
+ * @param regionInfo RegionInfo for default replica
+ * @param replicaId the replicaId for the HRegionLocation
+ * @return HRegionLocation parsed from the given meta row Result for the given replicaId
+ */
+ private static HRegionLocation getRegionLocation(final Result r, final HRegionInfo regionInfo,
+ final int replicaId) {
+ ServerName serverName = getServerName(r, replicaId);
+ long seqNum = getSeqNumDuringOpen(r, replicaId);
+ HRegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId);
+ return new HRegionLocation(replicaInfo, serverName, seqNum);
+ }
+
+ /**
+ * Returns HRegionInfo object from the column
+ * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
+ * table Result.
+ * @param data a Result object from the catalog table scan
+ * @return HRegionInfo or null
+ */
+ public static HRegionInfo getHRegionInfo(Result data) {
+ return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
+ }
+
+ /**
+ * Returns the HRegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and
+ * <code>qualifier</code> of the catalog table result.
+ * @param r a Result object from the catalog table scan
+ * @param qualifier Column family qualifier
+ * @return An HRegionInfo instance or null.
+ */
+ private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
+ Cell cell = r.getColumnLatestCell(getFamily(), qualifier);
+ if (cell == null) return null;
+ return HRegionInfo.parseFromOrNull(cell.getValueArray(),
+ cell.getValueOffset(), cell.getValueLength());
+ }
+
+ /**
+ * Returns the daughter regions by reading the corresponding columns of the catalog table
+ * Result.
+ * @param data a Result object from the catalog table scan
+ * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
+ * parent
+ */
+ public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
+ HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
+ HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
+
+ return new PairOfSameType<HRegionInfo>(splitA, splitB);
+ }
+
+ /**
+ * Returns the merge regions by reading the corresponding columns of the catalog table
+ * Result.
+ * @param data a Result object from the catalog table scan
+ * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
+ * parent
+ */
+ public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) throws IOException {
+ HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
+ HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
+
+ return new PairOfSameType<HRegionInfo>(mergeA, mergeB);
+ }
+
+ /**
* Implementations 'visit' a catalog table row.
*/
public interface Visitor {
Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java Thu Feb 6 02:04:53 2014
@@ -33,7 +33,6 @@ import java.util.concurrent.ConcurrentMa
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -49,7 +48,6 @@ import org.apache.hadoop.hbase.TableName
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
import org.cloudera.htrace.Trace;
import com.google.common.annotations.VisibleForTesting;
@@ -93,7 +91,7 @@ class AsyncProcess {
private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
protected static final AtomicLong COUNTER = new AtomicLong();
- /**
+ /**
* The context used to wait for results from one submit call.
* 1) If AsyncProcess is set to track errors globally, and not per call (for HTable puts),
* then errors and failed operations in this object will reflect global errors.
@@ -111,10 +109,15 @@ class AsyncProcess {
/** Return value from a submit that didn't contain any requests. */
private static final AsyncRequestFuture NO_REQS_RESULT = new AsyncRequestFuture() {
public final Object[] result = new Object[0];
+ @Override
public boolean hasError() { return false; }
+ @Override
public RetriesExhaustedWithDetailsException getErrors() { return null; }
+ @Override
public List<? extends Row> getFailedOperations() { return null; }
+ @Override
public Object[] getResults() { return result; }
+ @Override
public void waitUntilDone() throws InterruptedIOException {}
};
@@ -685,7 +688,7 @@ class AsyncProcess {
// Do not use the exception for updating cache because it might be coming from
// any of the regions in the MultiAction.
byte[] row = rsActions.actions.values().iterator().next().get(0).getAction().getRow();
- hConnection.updateCachedLocations(tableName, row, null, server);
+ hConnection.updateCachedLocations(tableName, null, row, null, server);
errorsByServer.reportServerError(server);
List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
@@ -790,7 +793,7 @@ class AsyncProcess {
if (!regionFailureRegistered) { // We're doing this once per location.
regionFailureRegistered = true;
// The location here is used as a server name.
- hConnection.updateCachedLocations(tableName, row.getRow(), result, server);
+ hConnection.updateCachedLocations(tableName, regionName, row.getRow(), result, server);
if (failureCount == 0) {
errorsByServer.reportServerError(server);
canRetry = errorsByServer.canRetryMore(numAttempt);
@@ -835,7 +838,7 @@ class AsyncProcess {
canRetry = errorsByServer.canRetryMore(numAttempt);
}
hConnection.updateCachedLocations(
- tableName, actions.get(0).getAction().getRow(), throwable, server);
+ tableName, region, actions.get(0).getAction().getRow(), throwable, server);
failureCount += actions.size();
for (Action<Row> action : actions) {
@@ -989,7 +992,7 @@ class AsyncProcess {
}
}
- /**
+ /**
* Only used w/useGlobalErrors ctor argument, for HTable backward compat.
* @return Whether there were any errors in any request since the last time
* {@link #waitForAllPreviousOpsAndReset(List)} was called, or AP was created.
Modified: hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=1565041&r1=1565040&r2=1565041&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/hbase-10070/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Thu Feb 6 02:04:53 2014
@@ -27,11 +27,11 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
@@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.protobuf.
* keeps a cache of locations and then knows how to re-calibrate after they move. You need one
* of these to talk to your HBase cluster. {@link HConnectionManager} manages instances of this
* class. See it for how to get one of these.
- *
+ *
* <p>This is NOT a connection to a particular server but to ALL servers in the cluster. Individual
* connections are managed at a lower level.
*
@@ -251,7 +251,9 @@ public interface HConnection extends Abo
* @return HRegionLocation that describes where to find the region in
* question
* @throws IOException if a remote or network exception occurs
+ * @deprecated This is no longer a public API
*/
+ @Deprecated
public HRegionLocation locateRegion(final TableName tableName,
final byte [] row) throws IOException;
@@ -305,11 +307,12 @@ public interface HConnection extends Abo
* Update the location cache. This is used internally by HBase, in most cases it should not be
* used by the client application.
* @param tableName the table name
+ * @param regionName the regionName
* @param rowkey the row
* @param exception the exception if any. Can be null.
* @param source the previous location
*/
- void updateCachedLocations(TableName tableName, byte[] rowkey,
+ void updateCachedLocations(TableName tableName, byte[] regionName, byte[] rowkey,
Object exception, ServerName source);
@Deprecated
@@ -345,7 +348,9 @@ public interface HConnection extends Abo
* regions from returned list.
* @return list of region locations for all regions of table
* @throws IOException
+ * @deprecated This is no longer a public API
*/
+ @Deprecated
public List<HRegionLocation> locateRegions(final TableName tableName,
final boolean useCache,
final boolean offlined) throws IOException;
@@ -388,6 +393,7 @@ public interface HConnection extends Abo
* @throws IOException if a remote or network exception occurs
* @deprecated You can pass master flag but nothing special is done.
*/
+ @Deprecated
AdminService.BlockingInterface getAdmin(final ServerName serverName, boolean getMaster)
throws IOException;
@@ -478,6 +484,7 @@ public interface HConnection extends Abo
* @throws IOException if a remote or network exception occurs
* @deprecated This method will be changed from public to package protected.
*/
+ @Deprecated
int getCurrentNrHRS() throws IOException;
/**