You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/06/14 07:40:35 UTC
svn commit: r1135385 [2/4] - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/catalog/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/io/ src/main/java/org/apache/hado...
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java?rev=1135385&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/migration/HRegionInfo090x.java Tue Jun 14 05:40:32 2011
@@ -0,0 +1,680 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.migration;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JenkinsHash;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.apache.hadoop.io.VersionedWritable;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * HRegion information.
+ * Contains HRegion id, start and end keys, a reference to this
+ * HRegions' table descriptor, etc.
+ */
+public class HRegionInfo090x extends VersionedWritable implements
+ WritableComparable<HRegionInfo090x>{
+ private static final byte VERSION = 0;
+ private static final Log LOG = LogFactory.getLog(HRegionInfo090x.class);
+
+ /**
+ * The new format for a region name contains its encodedName at the end.
+ * The encoded name also serves as the directory name for the region
+ * in the filesystem.
+ *
+ * New region name format:
+ * <tablename>,,<startkey>,<regionIdTimestamp>.<encodedName>.
+ * where,
+ * <encodedName> is a hex version of the MD5 hash of
+ * <tablename>,<startkey>,<regionIdTimestamp>
+ *
+ * The old region name format:
+ * <tablename>,<startkey>,<regionIdTimestamp>
+ * For region names in the old format, the encoded name is a 32-bit
+ * JenkinsHash integer value (in its decimal notation, string form).
+ *<p>
+ * **NOTE**
+ *
+ * ROOT, the first META region, and regions created by an older
+ * version of HBase (0.20 or prior) will continue to use the
+ * old region name format.
+ */
+
+ /** Separator used to demarcate the encodedName in a region name
+ * in the new format. See description on new format above.
+ */
+ private static final int ENC_SEPARATOR = '.';
+ public static final int MD5_HEX_LENGTH = 32;
+
+ /**
+ * Does region name contain its encoded name?
+ * @param regionName region name
+ * @return boolean indicating if this a new format region
+ * name which contains its encoded name.
+ */
+ private static boolean hasEncodedName(final byte[] regionName) {
+ // check if region name ends in ENC_SEPARATOR
+ if ((regionName.length >= 1)
+ && (regionName[regionName.length - 1] == ENC_SEPARATOR)) {
+ // region name is new format. it contains the encoded name.
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * @param regionName
+ * @return the encodedName
+ */
+ public static String encodeRegionName(final byte [] regionName) {
+ String encodedName;
+ if (hasEncodedName(regionName)) {
+ // region is in new format:
+ // <tableName>,<startKey>,<regionIdTimeStamp>/encodedName/
+ encodedName = Bytes.toString(regionName,
+ regionName.length - MD5_HEX_LENGTH - 1,
+ MD5_HEX_LENGTH);
+ } else {
+ // old format region name. ROOT and first META region also
+ // use this format.EncodedName is the JenkinsHash value.
+ int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName,
+ regionName.length, 0));
+ encodedName = String.valueOf(hashVal);
+ }
+ return encodedName;
+ }
+
+ /**
+ * Use logging.
+ * @param encodedRegionName The encoded regionname.
+ * @return <code>-ROOT-</code> if passed <code>70236052</code> or
+ * <code>.META.</code> if passed </code>1028785192</code> else returns
+ * <code>encodedRegionName</code>
+ */
+ public static String prettyPrint(final String encodedRegionName) {
+ if (encodedRegionName.equals("70236052")) {
+ return encodedRegionName + "/-ROOT-";
+ } else if (encodedRegionName.equals("1028785192")) {
+ return encodedRegionName + "/.META.";
+ }
+ return encodedRegionName;
+ }
+
+ /** delimiter used between portions of a region name */
+ public static final int DELIMITER = ',';
+
+ /** HRegionInfo for root region */
+ public static final HRegionInfo090x ROOT_REGIONINFO =
+ new HRegionInfo090x(0L, HTableDescriptor.ROOT_TABLEDESC);
+
+ /** HRegionInfo for first meta region */
+ public static final HRegionInfo090x FIRST_META_REGIONINFO =
+ new HRegionInfo090x(1L, HTableDescriptor.META_TABLEDESC);
+
+ private byte [] endKey = HConstants.EMPTY_BYTE_ARRAY;
+ // This flag is in the parent of a split while the parent is still referenced
+ // by daughter regions. We USED to set this flag when we disabled a table
+ // but now table state is kept up in zookeeper as of 0.90.0 HBase.
+ private boolean offLine = false;
+ private long regionId = -1;
+ private transient byte [] regionName = HConstants.EMPTY_BYTE_ARRAY;
+ private String regionNameStr = "";
+ private boolean split = false;
+ private byte [] startKey = HConstants.EMPTY_BYTE_ARRAY;
+ protected HTableDescriptor tableDesc = null;
+ private int hashCode = -1;
+ //TODO: Move NO_HASH to HStoreFile which is really the only place it is used.
+ public static final String NO_HASH = null;
+ private volatile String encodedName = NO_HASH;
+ private byte [] encodedNameAsBytes = null;
+
+ private void setHashCode() {
+ int result = Arrays.hashCode(this.regionName);
+ result ^= this.regionId;
+ result ^= Arrays.hashCode(this.startKey);
+ result ^= Arrays.hashCode(this.endKey);
+ result ^= Boolean.valueOf(this.offLine).hashCode();
+ result ^= this.tableDesc.hashCode();
+ this.hashCode = result;
+ }
+
+ /**
+ * Private constructor used constructing HRegionInfo for the catalog root and
+ * first meta regions
+ */
+ private HRegionInfo090x(long regionId, HTableDescriptor tableDesc) {
+ super();
+ this.regionId = regionId;
+ this.tableDesc = tableDesc;
+
+ // Note: Root & First Meta regions names are still in old format
+ this.regionName = createRegionName(tableDesc.getName(), null,
+ regionId, false);
+ this.regionNameStr = Bytes.toStringBinary(this.regionName);
+ setHashCode();
+ }
+
+ /** Default constructor - creates empty object */
+ public HRegionInfo090x() {
+ super();
+ this.tableDesc = new HTableDescriptor();
+ }
+
+ /**
+ * Construct HRegionInfo with explicit parameters
+ *
+ * @param tableDesc the table descriptor
+ * @param startKey first key in region
+ * @param endKey end of key range
+ * @throws IllegalArgumentException
+ */
+ public HRegionInfo090x(final HTableDescriptor tableDesc, final byte[] startKey,
+ final byte[] endKey)
+ throws IllegalArgumentException {
+ this(tableDesc, startKey, endKey, false);
+ }
+
+ /**
+ * Construct HRegionInfo with explicit parameters
+ *
+ * @param tableDesc the table descriptor
+ * @param startKey first key in region
+ * @param endKey end of key range
+ * @param split true if this region has split and we have daughter regions
+ * regions that may or may not hold references to this region.
+ * @throws IllegalArgumentException
+ */
+ public HRegionInfo090x(HTableDescriptor tableDesc, final byte[] startKey,
+ final byte[] endKey, final boolean split)
+ throws IllegalArgumentException {
+ this(tableDesc, startKey, endKey, split, System.currentTimeMillis());
+ }
+
+ /**
+ * Construct HRegionInfo with explicit parameters
+ *
+ * @param tableDesc the table descriptor
+ * @param startKey first key in region
+ * @param endKey end of key range
+ * @param split true if this region has split and we have daughter regions
+ * regions that may or may not hold references to this region.
+ * @param regionid Region id to use.
+ * @throws IllegalArgumentException
+ */
+ public HRegionInfo090x(HTableDescriptor tableDesc, final byte[] startKey,
+ final byte[] endKey, final boolean split, final long regionid)
+ throws IllegalArgumentException {
+ super();
+ if (tableDesc == null) {
+ throw new IllegalArgumentException("tableDesc cannot be null");
+ }
+ this.offLine = false;
+ this.regionId = regionid;
+ this.regionName = createRegionName(tableDesc.getName(), startKey, regionId, true);
+ this.regionNameStr = Bytes.toStringBinary(this.regionName);
+ this.split = split;
+ this.endKey = endKey == null? HConstants.EMPTY_END_ROW: endKey.clone();
+ this.startKey = startKey == null?
+ HConstants.EMPTY_START_ROW: startKey.clone();
+ this.tableDesc = tableDesc;
+ setHashCode();
+ }
+
+ /**
+ * Costruct a copy of another HRegionInfo
+ *
+ * @param other
+ */
+ public HRegionInfo090x(HRegionInfo090x other) {
+ super();
+ this.endKey = other.getEndKey();
+ this.offLine = other.isOffline();
+ this.regionId = other.getRegionId();
+ this.regionName = other.getRegionName();
+ this.regionNameStr = Bytes.toStringBinary(this.regionName);
+ this.split = other.isSplit();
+ this.startKey = other.getStartKey();
+ this.tableDesc = other.getTableDesc();
+ this.hashCode = other.hashCode();
+ this.encodedName = other.getEncodedName();
+ }
+
+ /**
+ * Make a region name of passed parameters.
+ * @param tableName
+ * @param startKey Can be null
+ * @param regionid Region id (Usually timestamp from when region was created).
+ * @param newFormat should we create the region name in the new format
+ * (such that it contains its encoded name?).
+ * @return Region name made of passed tableName, startKey and id
+ */
+ public static byte [] createRegionName(final byte [] tableName,
+ final byte [] startKey, final long regionid, boolean newFormat) {
+ return createRegionName(tableName, startKey, Long.toString(regionid), newFormat);
+ }
+
+ /**
+ * Make a region name of passed parameters.
+ * @param tableName
+ * @param startKey Can be null
+ * @param id Region id (Usually timestamp from when region was created).
+ * @param newFormat should we create the region name in the new format
+ * (such that it contains its encoded name?).
+ * @return Region name made of passed tableName, startKey and id
+ */
+ public static byte [] createRegionName(final byte [] tableName,
+ final byte [] startKey, final String id, boolean newFormat) {
+ return createRegionName(tableName, startKey, Bytes.toBytes(id), newFormat);
+ }
+
+ /**
+ * Make a region name of passed parameters.
+ * @param tableName
+ * @param startKey Can be null
+ * @param id Region id (Usually timestamp from when region was created).
+ * @param newFormat should we create the region name in the new format
+ * (such that it contains its encoded name?).
+ * @return Region name made of passed tableName, startKey and id
+ */
+ public static byte [] createRegionName(final byte [] tableName,
+ final byte [] startKey, final byte [] id, boolean newFormat) {
+ byte [] b = new byte [tableName.length + 2 + id.length +
+ (startKey == null? 0: startKey.length) +
+ (newFormat ? (MD5_HEX_LENGTH + 2) : 0)];
+
+ int offset = tableName.length;
+ System.arraycopy(tableName, 0, b, 0, offset);
+ b[offset++] = DELIMITER;
+ if (startKey != null && startKey.length > 0) {
+ System.arraycopy(startKey, 0, b, offset, startKey.length);
+ offset += startKey.length;
+ }
+ b[offset++] = DELIMITER;
+ System.arraycopy(id, 0, b, offset, id.length);
+ offset += id.length;
+
+ if (newFormat) {
+ //
+ // Encoded name should be built into the region name.
+ //
+ // Use the region name thus far (namely, <tablename>,<startKey>,<id>)
+ // to compute a MD5 hash to be used as the encoded name, and append
+ // it to the byte buffer.
+ //
+ String md5Hash = MD5Hash.getMD5AsHex(b, 0, offset);
+ byte [] md5HashBytes = Bytes.toBytes(md5Hash);
+
+ if (md5HashBytes.length != MD5_HEX_LENGTH) {
+ LOG.error("MD5-hash length mismatch: Expected=" + MD5_HEX_LENGTH +
+ "; Got=" + md5HashBytes.length);
+ }
+
+ // now append the bytes '.<encodedName>.' to the end
+ b[offset++] = ENC_SEPARATOR;
+ System.arraycopy(md5HashBytes, 0, b, offset, MD5_HEX_LENGTH);
+ offset += MD5_HEX_LENGTH;
+ b[offset++] = ENC_SEPARATOR;
+ }
+
+ return b;
+ }
+
+ /**
+ * Gets the table name from the specified region name.
+ * @param regionName
+ * @return Table name.
+ */
+ public static byte [] getTableName(byte [] regionName) {
+ int offset = -1;
+ for (int i = 0; i < regionName.length; i++) {
+ if (regionName[i] == DELIMITER) {
+ offset = i;
+ break;
+ }
+ }
+ byte [] tableName = new byte[offset];
+ System.arraycopy(regionName, 0, tableName, 0, offset);
+ return tableName;
+ }
+
+ /**
+ * Separate elements of a regionName.
+ * @param regionName
+ * @return Array of byte[] containing tableName, startKey and id
+ * @throws IOException
+ */
+ public static byte [][] parseRegionName(final byte [] regionName)
+ throws IOException {
+ int offset = -1;
+ for (int i = 0; i < regionName.length; i++) {
+ if (regionName[i] == DELIMITER) {
+ offset = i;
+ break;
+ }
+ }
+ if(offset == -1) throw new IOException("Invalid regionName format");
+ byte [] tableName = new byte[offset];
+ System.arraycopy(regionName, 0, tableName, 0, offset);
+ offset = -1;
+ for (int i = regionName.length - 1; i > 0; i--) {
+ if(regionName[i] == DELIMITER) {
+ offset = i;
+ break;
+ }
+ }
+ if(offset == -1) throw new IOException("Invalid regionName format");
+ byte [] startKey = HConstants.EMPTY_BYTE_ARRAY;
+ if(offset != tableName.length + 1) {
+ startKey = new byte[offset - tableName.length - 1];
+ System.arraycopy(regionName, tableName.length + 1, startKey, 0,
+ offset - tableName.length - 1);
+ }
+ byte [] id = new byte[regionName.length - offset - 1];
+ System.arraycopy(regionName, offset + 1, id, 0,
+ regionName.length - offset - 1);
+ byte [][] elements = new byte[3][];
+ elements[0] = tableName;
+ elements[1] = startKey;
+ elements[2] = id;
+ return elements;
+ }
+
+ /** @return the regionId */
+ public long getRegionId(){
+ return regionId;
+ }
+
+ /**
+ * @return the regionName as an array of bytes.
+ * @see #getRegionNameAsString()
+ */
+ public byte [] getRegionName(){
+ return regionName;
+ }
+
+ /**
+ * @return Region name as a String for use in logging, etc.
+ */
+ public String getRegionNameAsString() {
+ if (hasEncodedName(this.regionName)) {
+ // new format region names already have their encoded name.
+ return this.regionNameStr;
+ }
+
+ // old format. regionNameStr doesn't have the region name.
+ //
+ //
+ return this.regionNameStr + "." + this.getEncodedName();
+ }
+
+ /** @return the encoded region name */
+ public synchronized String getEncodedName() {
+ if (this.encodedName == NO_HASH) {
+ this.encodedName = encodeRegionName(this.regionName);
+ }
+ return this.encodedName;
+ }
+
+ public synchronized byte [] getEncodedNameAsBytes() {
+ if (this.encodedNameAsBytes == null) {
+ this.encodedNameAsBytes = Bytes.toBytes(getEncodedName());
+ }
+ return this.encodedNameAsBytes;
+ }
+
+ /** @return the startKey */
+ public byte [] getStartKey(){
+ return startKey;
+ }
+
+ /** @return the endKey */
+ public byte [] getEndKey(){
+ return endKey;
+ }
+
+ /**
+ * Returns true if the given inclusive range of rows is fully contained
+ * by this region. For example, if the region is foo,a,g and this is
+ * passed ["b","c"] or ["a","c"] it will return true, but if this is passed
+ * ["b","z"] it will return false.
+ * @throws IllegalArgumentException if the range passed is invalid (ie end < start)
+ */
+ public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) {
+ if (Bytes.compareTo(rangeStartKey, rangeEndKey) > 0) {
+ throw new IllegalArgumentException(
+ "Invalid range: " + Bytes.toStringBinary(rangeStartKey) +
+ " > " + Bytes.toStringBinary(rangeEndKey));
+ }
+
+ boolean firstKeyInRange = Bytes.compareTo(rangeStartKey, startKey) >= 0;
+ boolean lastKeyInRange =
+ Bytes.compareTo(rangeEndKey, endKey) < 0 ||
+ Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY);
+ return firstKeyInRange && lastKeyInRange;
+ }
+
+ /**
+ * Return true if the given row falls in this region.
+ */
+ public boolean containsRow(byte[] row) {
+ return Bytes.compareTo(row, startKey) >= 0 &&
+ (Bytes.compareTo(row, endKey) < 0 ||
+ Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY));
+ }
+
+ /** @return the tableDesc */
+ public HTableDescriptor getTableDesc(){
+ return tableDesc;
+ }
+
+ /**
+ * @param newDesc new table descriptor to use
+ */
+ public void setTableDesc(HTableDescriptor newDesc) {
+ this.tableDesc = newDesc;
+ }
+
+ /** @return true if this is the root region */
+ public boolean isRootRegion() {
+ return this.tableDesc.isRootRegion();
+ }
+
+ /** @return true if this region is from a table that is a meta table,
+ * either <code>.META.</code> or <code>-ROOT-</code>
+ */
+ public boolean isMetaTable() {
+ return this.tableDesc.isMetaTable();
+ }
+
+ /** @return true if this region is a meta region */
+ public boolean isMetaRegion() {
+ return this.tableDesc.isMetaRegion();
+ }
+
+ /**
+ * @return True if has been split and has daughters.
+ */
+ public boolean isSplit() {
+ return this.split;
+ }
+
+ /**
+ * @param split set split status
+ */
+ public void setSplit(boolean split) {
+ this.split = split;
+ }
+
+ /**
+ * @return True if this region is offline.
+ */
+ public boolean isOffline() {
+ return this.offLine;
+ }
+
+ /**
+ * The parent of a region split is offline while split daughters hold
+ * references to the parent. Offlined regions are closed.
+ * @param offLine Set online/offline status.
+ */
+ public void setOffline(boolean offLine) {
+ this.offLine = offLine;
+ }
+
+
+ /**
+ * @return True if this is a split parent region.
+ */
+ public boolean isSplitParent() {
+ if (!isSplit()) return false;
+ if (!isOffline()) {
+ LOG.warn("Region is split but NOT offline: " + getRegionNameAsString());
+ }
+ return true;
+ }
+
+ /**
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return "REGION => {" + HConstants.NAME + " => '" +
+ this.regionNameStr +
+ "', STARTKEY => '" +
+ Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" +
+ Bytes.toStringBinary(this.endKey) +
+ "', ENCODED => " + getEncodedName() + "," +
+ (isOffline()? " OFFLINE => true,": "") +
+ (isSplit()? " SPLIT => true,": "") +
+ " TABLE => {" + this.tableDesc.toString() + "}";
+ }
+
+ /**
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null) {
+ return false;
+ }
+ if (!(o instanceof HRegionInfo090x)) {
+ return false;
+ }
+ return this.compareTo((HRegionInfo090x)o) == 0;
+ }
+
+ /**
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ return this.hashCode;
+ }
+
+ /** @return the object version number */
+ @Override
+ public byte getVersion() {
+ return VERSION;
+ }
+
+ //
+ // Writable
+ //
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ Bytes.writeByteArray(out, endKey);
+ out.writeBoolean(offLine);
+ out.writeLong(regionId);
+ Bytes.writeByteArray(out, regionName);
+ out.writeBoolean(split);
+ Bytes.writeByteArray(out, startKey);
+ tableDesc.write(out);
+ out.writeInt(hashCode);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ this.endKey = Bytes.readByteArray(in);
+ this.offLine = in.readBoolean();
+ this.regionId = in.readLong();
+ this.regionName = Bytes.readByteArray(in);
+ this.regionNameStr = Bytes.toStringBinary(this.regionName);
+ this.split = in.readBoolean();
+ this.startKey = Bytes.readByteArray(in);
+ try {
+ this.tableDesc.readFields(in);
+ } catch(EOFException eofe) {
+ throw new IOException("HTD not found in input buffer");
+ }
+ this.hashCode = in.readInt();
+ }
+
+ //
+ // Comparable
+ //
+
+ public int compareTo(HRegionInfo090x o) {
+ if (o == null) {
+ return 1;
+ }
+
+ // Are regions of same table?
+ int result = Bytes.compareTo(this.tableDesc.getName(), o.tableDesc.getName());
+ if (result != 0) {
+ return result;
+ }
+
+ // Compare start keys.
+ result = Bytes.compareTo(this.startKey, o.startKey);
+ if (result != 0) {
+ return result;
+ }
+
+ // Compare end keys.
+ return Bytes.compareTo(this.endKey, o.endKey);
+ }
+
+ /**
+ * @return Comparator to use comparing {@link org.apache.hadoop.hbase.KeyValue}s.
+ */
+ public KVComparator getComparator() {
+ return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
+ KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
+ }
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Jun 14 05:40:32 2011
@@ -199,6 +199,8 @@ public class HRegion implements HeapSize
final Path regiondir;
KeyValue.KVComparator comparator;
+ private Pair<Long,Long> lastCompactInfo = null;
+
/*
* Data structure of write state flags used coordinating flushes,
* compactions and closes.
@@ -239,11 +241,11 @@ public class HRegion implements HeapSize
final WriteState writestate = new WriteState();
- final long memstoreFlushSize;
+ long memstoreFlushSize;
private volatile long lastFlushTime;
final RegionServerServices rsServices;
private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
- private final long blockingMemStoreSize;
+ private long blockingMemStoreSize;
final long threadWakeFrequency;
// Used to guard closes
final ReentrantReadWriteLock lock =
@@ -265,6 +267,7 @@ public class HRegion implements HeapSize
* Name of the region info file that resides just under the region directory.
*/
public final static String REGIONINFO_FILE = ".regioninfo";
+ private HTableDescriptor htableDescriptor = null;
/**
* Should only be used for testing purposes
@@ -319,14 +322,14 @@ public class HRegion implements HeapSize
10 * 1000);
String encodedNameStr = this.regionInfo.getEncodedName();
this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
- long flushSize = regionInfo.getTableDesc().getMemStoreFlushSize();
- if (flushSize == HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE) {
- flushSize = conf.getLong("hbase.hregion.memstore.flush.size",
- HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
- }
- this.memstoreFlushSize = flushSize;
- this.blockingMemStoreSize = this.memstoreFlushSize *
- conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
+ try {
+ LOG.info("Setting table desc from HDFS. Region = "
+ + this.regionInfo.getTableNameAsString());
+ loadHTableDescriptor(tableDir);
+ LOG.info(" This HTD from HDFS == " + this.htableDescriptor);
+ } catch (IOException ioe) {
+ LOG.error("Could not instantiate region as error loading HTableDescriptor");
+ }
// don't initialize coprocessors if not running within a regionserver
// TODO: revisit if coprocessors should load in other cases
if (rsServices != null) {
@@ -338,6 +341,40 @@ public class HRegion implements HeapSize
}
}
+ private void loadHTableDescriptor(Path tableDir) throws IOException {
+ LOG.debug("Assigning tabledesc from .tableinfo for region = "
+ + this.regionInfo.getRegionNameAsString());
+ // load HTableDescriptor
+ this.htableDescriptor = FSUtils.getTableDescriptor(tableDir, fs);
+
+ if (this.htableDescriptor != null) {
+ setHTableSpecificConf();
+ } else {
+ throw new IOException("Table description missing in " +
+ ".tableinfo. Cannot create new region."
+ + " current region is == " + this.regionInfo.toString());
+ }
+
+ }
+
+ private void setHTableSpecificConf() {
+ if (this.htableDescriptor != null) {
+ LOG.info("Setting up tabledescriptor config now ...");
+ long flushSize = this.htableDescriptor.getMemStoreFlushSize();
+ if (flushSize == HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE) {
+ flushSize = conf.getLong("hbase.hregion.memstore.flush.size",
+ HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
+ }
+ this.memstoreFlushSize = flushSize;
+ this.blockingMemStoreSize = this.memstoreFlushSize *
+ conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
+ }
+ }
+
+ public void setHtableDescriptor(HTableDescriptor htableDescriptor) {
+ this.htableDescriptor = htableDescriptor;
+ }
+
/**
* Initialize this region.
* @return What the next sequence (edit) id should be.
@@ -378,7 +415,7 @@ public class HRegion implements HeapSize
// Load in all the HStores. Get maximum seqid.
long maxSeqId = -1;
- for (HColumnDescriptor c : this.regionInfo.getTableDesc().getFamilies()) {
+ for (HColumnDescriptor c : this.htableDescriptor.getFamilies()) {
status.setStatus("Instantiating store for column family " + c);
Store store = instantiateHStore(this.tableDir, c);
this.stores.put(c.getName(), store);
@@ -398,7 +435,7 @@ public class HRegion implements HeapSize
SplitTransaction.cleanupAnySplitDetritus(this);
FSUtils.deleteDirectory(this.fs, new Path(regiondir, MERGEDIR));
- this.writestate.setReadOnly(this.regionInfo.getTableDesc().isReadOnly());
+ this.writestate.setReadOnly(this.htableDescriptor.isReadOnly());
this.writestate.compacting = 0;
this.lastFlushTime = EnvironmentEdgeManager.currentTimeMillis();
@@ -703,7 +740,7 @@ public class HRegion implements HeapSize
/** @return HTableDescriptor for this region */
public HTableDescriptor getTableDesc() {
- return this.regionInfo.getTableDesc();
+ return this.htableDescriptor;
}
/** @return HLog in use for this region */
@@ -1156,7 +1193,7 @@ public class HRegion implements HeapSize
// log-sequence-ids can be safely ignored.
if (wal != null) {
wal.completeCacheFlush(this.regionInfo.getEncodedNameAsBytes(),
- regionInfo.getTableDesc().getName(), completeSequenceId,
+ regionInfo.getTableName(), completeSequenceId,
this.getRegionInfo().isMetaRegion());
}
@@ -1268,7 +1305,7 @@ public class HRegion implements HeapSize
void prepareScanner(Scan scan) throws IOException {
if(!scan.hasFamilies()) {
// Adding all families to scanner
- for(byte[] family: regionInfo.getTableDesc().getFamiliesKeys()){
+ for(byte[] family: this.htableDescriptor.getFamiliesKeys()){
scan.addFamily(family);
}
}
@@ -1303,7 +1340,7 @@ public class HRegion implements HeapSize
private void prepareDelete(Delete delete) throws IOException {
// Check to see if this is a deleteRow insert
if(delete.getFamilyMap().isEmpty()){
- for(byte [] family : regionInfo.getTableDesc().getFamiliesKeys()){
+ for(byte [] family : this.htableDescriptor.getFamiliesKeys()){
// Don't eat the timestamp
delete.deleteFamily(family, delete.getTimeStamp());
}
@@ -1424,8 +1461,8 @@ public class HRegion implements HeapSize
// single WALEdit.
WALEdit walEdit = new WALEdit();
addFamilyMapToWALEdit(familyMap, walEdit);
- this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdit, now);
+ this.log.append(regionInfo, this.htableDescriptor.getName(),
+ walEdit, now, this.htableDescriptor);
}
// Now make changes to the memstore.
@@ -1683,8 +1720,8 @@ public class HRegion implements HeapSize
}
// Append the edit to WAL
- this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdit, now);
+ this.log.append(regionInfo, this.htableDescriptor.getName(),
+ walEdit, now, this.htableDescriptor);
// ------------------------------------
// STEP 4. Write back to memstore
@@ -1938,8 +1975,8 @@ public class HRegion implements HeapSize
if (writeToWAL) {
WALEdit walEdit = new WALEdit();
addFamilyMapToWALEdit(familyMap, walEdit);
- this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdit, now);
+ this.log.append(regionInfo, this.htableDescriptor.getName(),
+ walEdit, now, this.htableDescriptor);
}
long addedSize = applyFamilyMapToMemstore(familyMap);
@@ -2757,22 +2794,29 @@ public class HRegion implements HeapSize
* @param info Info for region to create.
* @param rootDir Root directory for HBase instance
* @param conf
+ * @param hTableDescriptor
* @return new HRegion
*
* @throws IOException
*/
public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
- final Configuration conf)
- throws IOException {
+ final Configuration conf,
+ final HTableDescriptor hTableDescriptor)
+ throws IOException {
+ LOG.info("creating HRegion " + info.getTableNameAsString()
+ + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
+ " Table name == " + info.getTableNameAsString());
+
Path tableDir =
- HTableDescriptor.getTableDir(rootDir, info.getTableDesc().getName());
+ HTableDescriptor.getTableDir(rootDir, info.getTableName());
Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
FileSystem fs = FileSystem.get(conf);
fs.mkdirs(regionDir);
+ FSUtils.createTableDescriptor(fs, hTableDescriptor, tableDir);
HRegion region = HRegion.newHRegion(tableDir,
- new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
- new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
- fs, conf, info, null);
+ new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
+ new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
+ fs, conf, info, null);
region.initialize();
return region;
}
@@ -2820,12 +2864,52 @@ public class HRegion implements HeapSize
throw new NullPointerException("Passed region info is null");
}
Path dir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
- info.getTableDesc().getName());
+ info.getTableName());
HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
rsServices);
return r.openHRegion(reporter);
}
+ public static HRegion openHRegion(Path tableDir, final HRegionInfo info,
+ final HLog wal, final Configuration conf)
+ throws IOException {
+ return openHRegion(tableDir, info, wal, conf, null, null);
+ }
+
+ /**
+ * Open a Region.
+ * @param tableDir Table directory
+ * @param info Info for region to be opened.
+ * @param wal HLog for region to use. This method will call
+ * HLog#setSequenceNumber(long) passing the result of the call to
+ * HRegion#getMinSequenceId() to ensure the log id is properly kept
+ * up. HRegionStore does this every time it opens a new region.
+ * @param conf
+ * @param reporter An interface we can report progress against.
+ * @return new HRegion
+ *
+ * @throws IOException
+ */
+ public static HRegion openHRegion(final Path tableDir, final HRegionInfo info,
+ final HLog wal, final Configuration conf,
+ final RegionServerServices rsServices,
+ final CancelableProgressable reporter)
+ throws IOException {
+ LOG.info("HRegion.openHRegion Region name ==" + info.getRegionNameAsString());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Opening region: " + info);
+ }
+ if (info == null) {
+ throw new NullPointerException("Passed region info is null");
+ }
+ Path dir = HTableDescriptor.getTableDir(tableDir,
+ info.getTableName());
+ HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
+ rsServices);
+ return r.openHRegion(reporter);
+ }
+
+
/**
* Open HRegion.
* Calls initialize and sets sequenceid.
@@ -2845,7 +2929,7 @@ public class HRegion implements HeapSize
}
private void checkCompressionCodecs() throws IOException {
- for (HColumnDescriptor fam: regionInfo.getTableDesc().getColumnFamilies()) {
+ for (HColumnDescriptor fam: this.htableDescriptor.getColumnFamilies()) {
CompressionTest.testCompression(fam.getCompression());
CompressionTest.testCompression(fam.getCompactionCompression());
}
@@ -2873,6 +2957,11 @@ public class HRegion implements HeapSize
HConstants.REGIONINFO_QUALIFIER,
EnvironmentEdgeManager.currentTimeMillis(),
Writables.getBytes(r.getRegionInfo())));
+ edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
+ HConstants.META_MIGRATION_QUALIFIER,
+ EnvironmentEdgeManager.currentTimeMillis(),
+ Bytes.toBytes("true")));
+
meta.put(HConstants.CATALOG_FAMILY, edits);
} finally {
meta.releaseRowLock(lid);
@@ -2911,7 +3000,7 @@ public class HRegion implements HeapSize
*/
public static Path getRegionDir(final Path rootdir, final HRegionInfo info) {
return new Path(
- HTableDescriptor.getTableDir(rootdir, info.getTableDesc().getName()),
+ HTableDescriptor.getTableDir(rootdir, info.getTableName()),
info.getEncodedName());
}
@@ -2989,8 +3078,8 @@ public class HRegion implements HeapSize
* @throws IOException
*/
public static HRegion merge(HRegion a, HRegion b) throws IOException {
- if (!a.getRegionInfo().getTableDesc().getNameAsString().equals(
- b.getRegionInfo().getTableDesc().getNameAsString())) {
+ if (!a.getRegionInfo().getTableNameAsString().equals(
+ b.getRegionInfo().getTableNameAsString())) {
throw new IOException("Regions do not belong to the same table");
}
@@ -3043,7 +3132,8 @@ public class HRegion implements HeapSize
? b.getEndKey()
: a.getEndKey());
- HRegionInfo newRegionInfo = new HRegionInfo(tabledesc, startKey, endKey);
+ HRegionInfo newRegionInfo =
+ new HRegionInfo(tabledesc.getName(), startKey, endKey);
LOG.info("Creating new region " + newRegionInfo.toString());
String encodedName = newRegionInfo.getEncodedName();
Path newRegionDir = HRegion.getRegionDir(a.getTableDir(), encodedName);
@@ -3182,7 +3272,7 @@ public class HRegion implements HeapSize
checkFamily(family);
}
} else { // Adding all families to scanner
- for (byte[] family: regionInfo.getTableDesc().getFamiliesKeys()) {
+ for (byte[] family: this.htableDescriptor.getFamiliesKeys()) {
get.addFamily(family);
}
}
@@ -3388,8 +3478,8 @@ public class HRegion implements HeapSize
// Actually write to WAL now
if (writeToWAL) {
- this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdits, now);
+ this.log.append(regionInfo, this.htableDescriptor.getName(),
+ walEdits, now, this.htableDescriptor);
}
size = this.addAndGetGlobalMemstoreSize(size);
@@ -3459,8 +3549,8 @@ public class HRegion implements HeapSize
long now = EnvironmentEdgeManager.currentTimeMillis();
WALEdit walEdit = new WALEdit();
walEdit.add(newKv);
- this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
- walEdit, now);
+ this.log.append(regionInfo, this.htableDescriptor.getName(),
+ walEdit, now, this.htableDescriptor);
}
// Now request the ICV to the store, this will set the timestamp
@@ -3493,19 +3583,17 @@ public class HRegion implements HeapSize
private void checkFamily(final byte [] family)
throws NoSuchColumnFamilyException {
- if(!regionInfo.getTableDesc().hasFamily(family)) {
+ if (!this.htableDescriptor.hasFamily(family)) {
throw new NoSuchColumnFamilyException("Column family " +
- Bytes.toStringBinary(family) + " does not exist in region " + this
- + " in table " + regionInfo.getTableDesc());
- }
+ Bytes.toString(family) + " does not exist in region " + this
+ + " in table " + this.htableDescriptor);
+ }
}
public static final long FIXED_OVERHEAD = ClassSize.align(
- ClassSize.OBJECT + // this
- (4 * Bytes.SIZEOF_LONG) + // memstoreFlushSize, lastFlushTime, blockingMemStoreSize, threadWakeFrequency
- Bytes.SIZEOF_BOOLEAN + // splitRequest
- ClassSize.ARRAY + // splitPoint
- (26 * ClassSize.REFERENCE));
+ (4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
+ ClassSize.align(28 * ClassSize.REFERENCE) + ClassSize.OBJECT +
+ ClassSize.align(Bytes.SIZEOF_INT));
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
ClassSize.OBJECT + // closeLock
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Jun 14 05:40:32 2011
@@ -1348,6 +1348,7 @@ public class HRegionServer implements HR
public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct,
final boolean daughter)
throws KeeperException, IOException {
+ LOG.info("HRS.PostOpenDeployTasks");
// Do checks to see if we need to compact (references or too many files)
for (Store s : r.getStores().values()) {
if (s.hasReferences() || s.needsCompaction()) {
@@ -1357,24 +1358,36 @@ public class HRegionServer implements HR
// Add to online regions if all above was successful.
addToOnlineRegions(r);
-
+ LOG.info("addToOnlineRegions is done" + r.getRegionInfo());
// Update ZK, ROOT or META
if (r.getRegionInfo().isRootRegion()) {
+
+ LOG.info("setRootLocation");
RootLocationEditor.setRootLocation(getZooKeeper(),
this.serverNameFromMasterPOV);
} else if (r.getRegionInfo().isMetaRegion()) {
+ LOG.info("updateMetaLocation");
+
MetaEditor.updateMetaLocation(ct, r.getRegionInfo(),
this.serverNameFromMasterPOV);
} else {
+ LOG.info("updateMetaLocation 111");
+
if (daughter) {
+ LOG.info("updateMetaLocation 22");
+
// If daughter of a split, update whole row, not just location.
MetaEditor.addDaughter(ct, r.getRegionInfo(),
this.serverNameFromMasterPOV);
} else {
+ LOG.info("updateMetaLocation 33");
+
MetaEditor.updateRegionLocation(ct, r.getRegionInfo(),
this.serverNameFromMasterPOV);
}
}
+ LOG.info("END HRS.PostOpenDeployTasks");
+
}
/**
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java Tue Jun 14 05:40:32 2011
@@ -22,10 +22,7 @@ package org.apache.hadoop.hbase.regionse
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
-import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
@@ -166,6 +163,12 @@ class LogRoller extends Thread implement
}
@Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey,
+ WALEdit logEdit) {
+ //Not interested
+ }
+
+ @Override
public void logCloseRequested() {
// not interested
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java Tue Jun 14 05:40:32 2011
@@ -170,9 +170,9 @@ public class SplitTransaction {
return false;
}
long rid = getDaughterRegionIdTimestamp(hri);
- this.hri_a = new HRegionInfo(hri.getTableDesc(), startKey, this.splitrow,
+ this.hri_a = new HRegionInfo(hri.getTableName(), startKey, this.splitrow,
false, rid);
- this.hri_b = new HRegionInfo(hri.getTableDesc(), this.splitrow, endKey,
+ this.hri_b = new HRegionInfo(hri.getTableName(), this.splitrow, endKey,
false, rid);
return true;
}
@@ -764,4 +764,4 @@ public class SplitTransaction {
return ZKAssign.transitionNode(zkw, parent, serverName,
EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
}
-}
\ No newline at end of file
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Jun 14 05:40:32 2011
@@ -39,11 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -195,9 +191,15 @@ public class Store implements HeapSize {
// Check if this is in-memory store
this.inMemory = family.isInMemory();
+ long maxFileSize = 0L;
+ HTableDescriptor hTableDescriptor = region.getTableDesc();
+ if (hTableDescriptor != null) {
+ maxFileSize = hTableDescriptor.getMaxFileSize();
+ } else {
+ maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE;
+ }
// By default we split region if a file > HConstants.DEFAULT_MAX_FILE_SIZE.
- long maxFileSize = info.getTableDesc().getMaxFileSize();
if (maxFileSize == HConstants.DEFAULT_MAX_FILE_SIZE) {
maxFileSize = conf.getLong("hbase.hregion.max.filesize",
HConstants.DEFAULT_MAX_FILE_SIZE);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java Tue Jun 14 05:40:32 2011
@@ -24,6 +24,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.executor.EventHandler;
@@ -71,9 +72,7 @@ public class OpenRegionHandler extends E
public void process() throws IOException {
try {
final String name = regionInfo.getRegionNameAsString();
- LOG.debug("Processing open of " + name);
if (this.server.isStopped() || this.rsServices.isStopping()) {
- LOG.info("Server stopping or stopped, skipping open of " + name);
return;
}
final String encodedName = regionInfo.getEncodedName();
@@ -182,6 +181,7 @@ public class OpenRegionHandler extends E
Thread.currentThread().interrupt();
}
}
+
// Was there an exception opening the region? This should trigger on
// InterruptedException too. If so, we failed.
return !t.interrupted() && t.getException() == null;
@@ -264,6 +264,33 @@ public class OpenRegionHandler extends E
/**
* @return Instance of HRegion if successful open else null.
*/
+ HRegion openRegion(Path tableDir) {
+ HRegion region = null;
+ try {
+ // Instantiate the region. This also periodically tickles our zk OPENING
+ // state so master doesn't timeout this region in transition.
+ region = HRegion.openHRegion(tableDir, this.regionInfo, this.rsServices.getWAL(),
+ this.server.getConfiguration(), this.rsServices,
+ new CancelableProgressable() {
+ public boolean progress() {
+ // We may lose the znode ownership during the open. Currently its
+ // too hard interrupting ongoing region open. Just let it complete
+ // and check we still have the znode after region open.
+ return tickleOpening("open_region_progress");
+ }
+ });
+ } catch (IOException e) {
+ // We failed open. Let our znode expire in regions-in-transition and
+ // Master will assign elsewhere. Presumes nothing to close.
+ LOG.error("Failed open of region=" +
+ this.regionInfo.getRegionNameAsString(), e);
+ }
+ return region;
+ }
+
+ /**
+ * @return Instance of HRegion if successful open else null.
+ */
HRegion openRegion() {
HRegion region = null;
try {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Jun 14 05:40:32 2011
@@ -54,11 +54,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -832,22 +828,6 @@ public class HLog implements Syncable {
}
}
- /** Append an entry to the log.
- *
- * @param regionInfo
- * @param logEdit
- * @param now Time of this edit write.
- * @throws IOException
- */
- public void append(HRegionInfo regionInfo, WALEdit logEdit,
- final long now,
- final boolean isMetaRegion)
- throws IOException {
- byte [] regionName = regionInfo.getEncodedNameAsBytes();
- byte [] tableName = regionInfo.getTableDesc().getName();
- this.append(regionInfo, makeKey(regionName, tableName, -1, now), logEdit);
- }
-
/**
* @param now
* @param regionName
@@ -867,7 +847,8 @@ public class HLog implements Syncable {
* @param logKey
* @throws IOException
*/
- public void append(HRegionInfo regionInfo, HLogKey logKey, WALEdit logEdit)
+ public void append(HRegionInfo regionInfo, HLogKey logKey, WALEdit logEdit,
+ HTableDescriptor htd)
throws IOException {
if (this.closed) {
throw new IOException("Cannot append; log is closed");
@@ -882,14 +863,14 @@ public class HLog implements Syncable {
// is greater than or equal to the value in lastSeqWritten.
this.lastSeqWritten.putIfAbsent(regionInfo.getEncodedNameAsBytes(),
Long.valueOf(seqNum));
- doWrite(regionInfo, logKey, logEdit);
+ doWrite(regionInfo, logKey, logEdit, htd);
this.numEntries.incrementAndGet();
}
// Sync if catalog region, and if not then check if that table supports
// deferred log flushing
if (regionInfo.isMetaRegion() ||
- !regionInfo.getTableDesc().isDeferredLogFlush()) {
+ !htd.isDeferredLogFlush()) {
// sync txn to file system
this.sync();
}
@@ -919,7 +900,7 @@ public class HLog implements Syncable {
* @throws IOException
*/
public void append(HRegionInfo info, byte [] tableName, WALEdit edits,
- final long now)
+ final long now, HTableDescriptor htd)
throws IOException {
if (edits.isEmpty()) return;
if (this.closed) {
@@ -937,13 +918,13 @@ public class HLog implements Syncable {
byte [] hriKey = info.getEncodedNameAsBytes();
this.lastSeqWritten.putIfAbsent(hriKey, seqNum);
HLogKey logKey = makeKey(hriKey, tableName, seqNum, now);
- doWrite(info, logKey, edits);
+ doWrite(info, logKey, edits, htd);
this.numEntries.incrementAndGet();
}
// Sync if catalog region, and if not then check if that table supports
// deferred log flushing
if (info.isMetaRegion() ||
- !info.getTableDesc().isDeferredLogFlush()) {
+ !htd.isDeferredLogFlush()) {
// sync txn to file system
this.sync();
}
@@ -1077,14 +1058,15 @@ public class HLog implements Syncable {
}
}
- protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
+ protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit,
+ HTableDescriptor htd)
throws IOException {
if (!this.enabled) {
return;
}
if (!this.listeners.isEmpty()) {
for (WALObserver i: this.listeners) {
- i.visitLogEntryBeforeWrite(info, logKey, logEdit);
+ i.visitLogEntryBeforeWrite(htd, logKey, logEdit);
}
}
try {
@@ -1100,12 +1082,12 @@ public class HLog implements Syncable {
writeOps++;
if (took > 1000) {
long len = 0;
- for(KeyValue kv : logEdit.getKeyValues()) {
- len += kv.getLength();
+ for(KeyValue kv : logEdit.getKeyValues()) {
+ len += kv.getLength();
}
LOG.warn(String.format(
"%s took %d ms appending an edit to hlog; editcount=%d, len~=%s",
- Thread.currentThread().getName(), took, this.numEntries.get(),
+ Thread.currentThread().getName(), took, this.numEntries.get(),
StringUtils.humanReadableInt(len)));
}
} catch (IOException e) {
@@ -1115,6 +1097,7 @@ public class HLog implements Syncable {
}
}
+
/** @return How many items have been added to the log */
int getNumEntries() {
return numEntries.get();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALObserver.java Tue Jun 14 05:40:32 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionse
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
/**
* Get notification of {@link HLog}/WAL log events. The invocations are inline
@@ -51,4 +52,14 @@ public interface WALObserver {
*/
public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
WALEdit logEdit);
+
+ /**
+ *
+ * @param htd
+ * @param logKey
+ * @param logEdit
+ */
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey,
+ WALEdit logEdit);
+
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java Tue Jun 14 05:40:32 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
@@ -132,12 +133,18 @@ public class Replication implements WALO
@Override
public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey,
WALEdit logEdit) {
+ // Not interested
+ }
+
+ @Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey,
+ WALEdit logEdit) {
NavigableMap<byte[], Integer> scopes =
new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
byte[] family;
for (KeyValue kv : logEdit.getKeyValues()) {
family = kv.getFamily();
- int scope = info.getTableDesc().getFamily(family).getScope();
+ int scope = htd.getFamily(family).getScope();
if (scope != REPLICATION_SCOPE_LOCAL &&
!scopes.containsKey(family)) {
scopes.put(family, scope);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Tue Jun 14 05:40:32 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -833,4 +834,225 @@ public class FSUtils {
}
LOG.info("Finished lease recover attempt for " + p);
}
+
+
+ public static Map<String, HTableDescriptor> getTableDescriptors(
+ final Configuration config)
+ throws IOException {
+ Path path = getRootDir(config);
+ // since HMaster.getFileSystem() is package private
+ FileSystem fs = path.getFileSystem(config);
+ return getTableDescriptors(fs, path);
+ }
+
+ public static Map<String, HTableDescriptor> getTableDescriptors(
+ final FileSystem fs, final Path hbaseRootDir)
+ throws IOException {
+ Map<String, HTableDescriptor> desc =
+ new HashMap<String, HTableDescriptor>();
+ DirFilter df = new DirFilter(fs);
+ // presumes any directory under hbase.rootdir is a table
+ FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
+ for (FileStatus tableDir : tableDirs) {
+ Path d = tableDir.getPath();
+ String tableName = d.getName();
+
+ if (tableName.equals(HConstants.HREGION_LOGDIR_NAME)
+ || tableName.equals(Bytes.toString(HConstants.ROOT_TABLE_NAME))
+ || tableName.equals(Bytes.toString(HConstants.META_TABLE_NAME))
+ || tableName.equals(HConstants.HREGION_OLDLOGDIR_NAME)
+ ) {
+ continue;
+ }
+ LOG.info("Adding tabledescriptor for table = " + tableName);
+ HTableDescriptor htd = readTableDescriptor(fs, hbaseRootDir,
+ tableName);
+ if (htd != null) {
+ if (!desc.containsKey(tableName)) {
+ desc.put(tableName, htd);
+ }
+ }
+ }
+ return desc;
+ }
+
+ private static Path getTableInfoPath(Path hbaseRootDir, String tableName) {
+ Path tablePath = new Path(hbaseRootDir, tableName);
+ return new Path(tablePath, HConstants.TABLEINFO_NAME);
+ }
+
+ /**
+ * Get table info path for a table.
+ * @param tableName
+ * @return Table info path
+ */
+ private static Path getTableInfoPath(byte[] tableName, Configuration conf) throws IOException {
+ Path tablePath = new Path(getRootDir(conf), Bytes.toString(tableName));
+ Path tableInfoPath = new Path(tablePath, HConstants.TABLEINFO_NAME);
+ return tableInfoPath;
+ }
+
+ private static Path getTablePath(byte[] tableName, Configuration conf) throws IOException {
+ return new Path(getRootDir(conf), Bytes.toString(tableName));
+ }
+
+ private static FileSystem getCurrentFileSystem(Configuration conf) throws IOException {
+ return getRootDir(conf).getFileSystem(conf);
+ }
+
+ /**
+ * Get HTableDescriptor
+ * @param config
+ * @param tableName
+ * @return HTableDescriptor for table
+ * @throws IOException
+ */
+ public static HTableDescriptor getHTableDescriptor(Configuration config,
+ String tableName)
+ throws IOException {
+ Path path = getRootDir(config);
+ FileSystem fs = path.getFileSystem(config);
+ return readTableDescriptor(fs, path, tableName);
+ }
+
+ private static HTableDescriptor readTableDescriptor(FileSystem fs,
+ Path hbaseRootDir,
+ String tableName) {
+ try {
+ FSDataInputStream fsDataInputStream =
+ fs.open(getTableInfoPath(hbaseRootDir, tableName));
+ HTableDescriptor hTableDescriptor = new HTableDescriptor();
+ hTableDescriptor.readFields(fsDataInputStream);
+ fsDataInputStream.close();
+ return hTableDescriptor;
+ } catch (IOException ioe) {
+ LOG.info("Exception during readTableDecriptor. Current table name = " + tableName , ioe);
+ }
+ return null;
+ }
+
+ /**
+ * Get HTD from HDFS.
+ * @param fs
+ * @param hbaseRootDir
+ * @param tableName
+ * @return
+ * @throws IOException
+ */
+ public static HTableDescriptor getTableDescriptor(FileSystem fs,
+ Path hbaseRootDir,
+ byte[] tableName)
+ throws IOException {
+ return readTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
+ }
+
+
+ public static HTableDescriptor getTableDescriptor(Path tableDir, FileSystem fs) {
+ try {
+ LOG.info("Reading table descriptor from .tableinfo. current path = "
+ + tableDir);
+ if (tableDir == null) {
+ LOG.info("Reading table descriptor from .tableinfo current tablename is NULL ");
+ return null;
+ }
+
+ FSDataInputStream fsDataInputStream =
+ fs.open(new Path(tableDir, HConstants.TABLEINFO_NAME));
+ HTableDescriptor hTableDescriptor = new HTableDescriptor();
+ hTableDescriptor.readFields(fsDataInputStream);
+ LOG.info("Current tabledescriptor from .tableinfo is " + hTableDescriptor.toString());
+ fsDataInputStream.close();
+ return hTableDescriptor;
+ } catch (IOException ioe) {
+ LOG.info("Exception during getTableDescriptor ", ioe);
+ }
+ return null;
+ }
+
+ /**
+ * Create new HTableDescriptor in HDFS.
+ * @param htableDescriptor
+ */
+ public static void createTableDescriptor(HTableDescriptor htableDescriptor,
+ Configuration conf) {
+ try {
+ Path tableDir = getTablePath(htableDescriptor.getName(), conf);
+ FileSystem fs = getCurrentFileSystem(conf);
+ createTableDescriptor(fs, htableDescriptor, tableDir);
+ } catch(IOException ioe) {
+ LOG.info("IOException while trying to create tableInfo in HDFS", ioe);
+ }
+ }
+
+ public static void createTableDescriptor(FileSystem fs,
+ HTableDescriptor htableDescriptor,
+ Path tableDir) {
+ try {
+ Path tableInfoPath = new Path(tableDir, HConstants.TABLEINFO_NAME);
+ LOG.info("Current tableInfoPath = " + tableInfoPath
+ + " tableDir = " + tableDir) ;
+ if (fs.exists(tableInfoPath) &&
+ fs.getFileStatus(tableInfoPath).getLen() > 0) {
+ LOG.info("TableInfo already exists.. Skipping creation");
+ return;
+ }
+ writeTableDescriptor(fs, htableDescriptor, tableDir);
+ } catch(IOException ioe) {
+ LOG.info("IOException while trying to create tableInfo in HDFS", ioe);
+ }
+ }
+
+ private static void writeTableDescriptor(FileSystem fs,
+ HTableDescriptor hTableDescriptor,
+ Path tableDir) throws IOException {
+ // Create in tmpdir and then move into place in case we crash after
+ // create but before close. If we don't successfully close the file,
+ // subsequent region reopens will fail the below because create is
+ // registered in NN.
+ Path tableInfoPath = new Path(tableDir, HConstants.TABLEINFO_NAME);
+ Path tmpPath = new Path(new Path(tableDir,".tmp"),
+ HConstants.TABLEINFO_NAME);
+ LOG.info("TableInfoPath = " + tableInfoPath + " tmpPath = " + tmpPath);
+ FSDataOutputStream out = fs.create(tmpPath, true);
+ try {
+ hTableDescriptor.write(out);
+ out.write('\n');
+ out.write('\n');
+ out.write(Bytes.toBytes(hTableDescriptor.toString()));
+ } finally {
+ out.close();
+ }
+ if (!fs.rename(tmpPath, tableInfoPath)) {
+ throw new IOException("Unable to rename " + tmpPath + " to " +
+ tableInfoPath);
+ } else {
+ LOG.info("TableDescriptor stored. TableInfoPath = " + tableInfoPath);
+ }
+ }
+
+
+ public static void updateHTableDescriptor(FileSystem fs,
+ Configuration conf,
+ HTableDescriptor hTableDescriptor) throws IOException
+ {
+ Path tableInfoPath = getTableInfoPath(hTableDescriptor.getName(), conf);
+ FSDataOutputStream out = fs.create(tableInfoPath, true);
+ try {
+ hTableDescriptor.write(out);
+ out.write('\n');
+ out.write('\n');
+ out.write(Bytes.toBytes(hTableDescriptor.toString()));
+ LOG.info("updateHTableDescriptor. Updated tableinfo in HDFS under "
+ + tableInfoPath + " For HTD => "
+ + hTableDescriptor.toString());
+ } finally {
+ out.close();
+ }
+ }
+
+ private static Path getTmpDir(HTableDescriptor htableDescriptor, Configuration configuration)
+ throws IOException {
+ return new Path(getTablePath(htableDescriptor.getName(), configuration), ".tmp");
+ }
+
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Tue Jun 14 05:40:32 2011
@@ -156,19 +156,24 @@ public class HBaseFsck {
// get a list of all tables that have not changed recently.
AtomicInteger numSkipped = new AtomicInteger(0);
HTableDescriptor[] allTables = getTables(numSkipped);
- errors.print("Number of Tables: " + allTables.length);
+ errors.print("Number of Tables: " +
+ (allTables == null ? 0 : allTables.length));
if (details) {
if (numSkipped.get() > 0) {
errors.detail("Number of Tables in flux: " + numSkipped.get());
}
- for (HTableDescriptor td : allTables) {
+ if (allTables != null && allTables.length > 0) {
+ for (HTableDescriptor td : allTables) {
String tableName = td.getNameAsString();
errors.detail(" Table: " + tableName + "\t" +
(td.isReadOnly() ? "ro" : "rw") + "\t" +
(td.isRootRegion() ? "ROOT" :
(td.isMetaRegion() ? "META" : " ")) + "\t" +
" families: " + td.getFamilies().size());
+ }
+
}
+
}
// From the master, get a list of all known live region servers
@@ -255,7 +260,7 @@ public class HBaseFsck {
* @throws KeeperException
*/
private boolean isTableDisabled(HRegionInfo regionInfo) {
- return disabledTables.contains(regionInfo.getTableDesc().getName());
+ return disabledTables.contains(regionInfo.getTableName());
}
/**
@@ -521,7 +526,7 @@ public class HBaseFsck {
if (hbi.deployedOn.size() == 0) continue;
// We should be safe here
- String tableName = hbi.metaEntry.getTableDesc().getNameAsString();
+ String tableName = hbi.metaEntry.getTableNameAsString();
TInfo modTInfo = tablesInfo.get(tableName);
if (modTInfo == null) {
modTInfo = new TInfo(tableName);
@@ -652,8 +657,8 @@ public class HBaseFsck {
* @return tables that have not been modified recently
* @throws IOException if an error is encountered
*/
- HTableDescriptor[] getTables(AtomicInteger numSkipped) {
- TreeSet<HTableDescriptor> uniqueTables = new TreeSet<HTableDescriptor>();
+ HTableDescriptor[] getTables(AtomicInteger numSkipped) {
+ List<String> tableNames = new ArrayList<String>();
long now = System.currentTimeMillis();
for (HbckInfo hbi : regionInfo.values()) {
@@ -663,15 +668,27 @@ public class HBaseFsck {
// pick only those tables that were not modified in the last few milliseconds.
if (info != null && info.getStartKey().length == 0 && !info.isMetaRegion()) {
if (info.modTime + timelag < now) {
- uniqueTables.add(info.getTableDesc());
+ tableNames.add(info.getTableNameAsString());
} else {
numSkipped.incrementAndGet(); // one more in-flux table
}
}
}
- return uniqueTables.toArray(new HTableDescriptor[uniqueTables.size()]);
+ return getHTableDescriptors(tableNames);
}
+ HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
+ HTableDescriptor[] htd = null;
+ try {
+ LOG.info("getHTableDescriptors == tableNames => " + tableNames);
+ htd = new HBaseAdmin(conf).getTableDescriptors(tableNames);
+ } catch (IOException e) {
+ LOG.debug("Exception getting table descriptors", e);
+ }
+ return htd;
+ }
+
+
/**
* Gets the entry in regionInfo corresponding to the the given encoded
* region name. If the region has not been seen yet, a new entry is added
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java Tue Jun 14 05:40:32 2011
@@ -261,7 +261,7 @@ class HMerge {
Bytes.toString(HConstants.REGIONINFO_QUALIFIER));
}
HRegionInfo region = Writables.getHRegionInfo(regionInfoValue);
- if (!Bytes.equals(region.getTableDesc().getName(), this.tableName)) {
+ if (!Bytes.equals(region.getTableName(), this.tableName)) {
return null;
}
return region;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java Tue Jun 14 05:40:32 2011
@@ -322,78 +322,6 @@ public class MetaUtils {
}
/**
- * Offline version of the online TableOperation,
- * org.apache.hadoop.hbase.master.AddColumn.
- * @param tableName table name
- * @param hcd Add this column to <code>tableName</code>
- * @throws IOException e
- */
- public void addColumn(final byte [] tableName,
- final HColumnDescriptor hcd)
- throws IOException {
- List<HRegionInfo> metas = getMETARows(tableName);
- for (HRegionInfo hri: metas) {
- final HRegion m = getMetaRegion(hri);
- scanMetaRegion(m, new ScannerListener() {
- private boolean inTable = true;
-
- @SuppressWarnings("synthetic-access")
- public boolean processRow(HRegionInfo info) throws IOException {
- LOG.debug("Testing " + Bytes.toString(tableName) + " against " +
- Bytes.toString(info.getTableDesc().getName()));
- if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
- this.inTable = false;
- info.getTableDesc().addFamily(hcd);
- updateMETARegionInfo(m, info);
- return true;
- }
- // If we got here and we have not yet encountered the table yet,
- // inTable will be false. Otherwise, we've passed out the table.
- // Stop the scanner.
- return this.inTable;
- }});
- }
- }
-
- /**
- * Offline version of the online TableOperation,
- * org.apache.hadoop.hbase.master.DeleteColumn.
- * @param tableName table name
- * @param columnFamily Name of column name to remove.
- * @throws IOException e
- */
- public void deleteColumn(final byte [] tableName,
- final byte [] columnFamily) throws IOException {
- List<HRegionInfo> metas = getMETARows(tableName);
- for (HRegionInfo hri: metas) {
- final HRegion m = getMetaRegion(hri);
- scanMetaRegion(m, new ScannerListener() {
- private boolean inTable = true;
-
- @SuppressWarnings("synthetic-access")
- public boolean processRow(HRegionInfo info) throws IOException {
- if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
- this.inTable = false;
- info.getTableDesc().removeFamily(columnFamily);
- updateMETARegionInfo(m, info);
- Path tabledir = new Path(rootdir,
- info.getTableDesc().getNameAsString());
- Path p = Store.getStoreHomedir(tabledir, info.getEncodedName(),
- columnFamily);
- if (!fs.delete(p, true)) {
- LOG.warn("Failed delete of " + p);
- }
- return false;
- }
- // If we got here and we have not yet encountered the table yet,
- // inTable will be false. Otherwise, we've passed out the table.
- // Stop the scanner.
- return this.inTable;
- }});
- }
- }
-
- /**
* Update COL_REGIONINFO in meta region r with HRegionInfo hri
*
* @param r region
@@ -466,7 +394,7 @@ public class MetaUtils {
public boolean processRow(HRegionInfo info) throws IOException {
SL_LOG.debug("Testing " + info);
- if (Bytes.equals(info.getTableDesc().getName(),
+ if (Bytes.equals(info.getTableName(),
HConstants.META_TABLE_NAME)) {
result.add(info);
return false;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java Tue Jun 14 05:40:32 2011
@@ -560,8 +560,9 @@ public class RegionSplitter {
if (sk.length == 0)
sk = splitAlgo.firstRow();
String startKey = splitAlgo.rowToStr(sk);
+ HTableDescriptor htd = table.getTableDescriptor();
// check every Column Family for that region
- for (HColumnDescriptor c : hri.getTableDesc().getFamilies()) {
+ for (HColumnDescriptor c : htd.getFamilies()) {
Path cfDir = Store.getStoreHomedir(tableDir, hri.getEncodedName(),
c.getName());
if (fs.exists(cfDir)) {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Writables.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Writables.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Writables.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Writables.java Tue Jun 14 05:40:32 2011
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.util;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.migration.HRegionInfo090x;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Writable;
@@ -62,7 +63,7 @@ public class Writables {
/**
* Put a bunch of Writables as bytes all into the one byte array.
- * @param w writable
+ * @param ws writable
* @return The bytes of <code>w</code> gotten by running its
* {@link Writable#write(java.io.DataOutput)} method.
* @throws IOException e
@@ -215,4 +216,16 @@ public class Writables {
}
return tgt;
}
+
+ /**
+ * Get HREgionInfoForMigration serialized from bytes.
+ * @param bytes serialized bytes
+ * @return HRegionInfoForMigration
+ * @throws IOException
+ */
+ public static HRegionInfo090x getHRegionInfoForMigration(final byte [] bytes)
+ throws IOException {
+ return (HRegionInfo090x)getWritable(bytes, new HRegionInfo090x());
+ }
+
}
\ No newline at end of file
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Tue Jun 14 05:40:32 2011
@@ -57,9 +57,15 @@ public abstract class HBaseTestCase exte
/** configuration parameter name for test directory */
public static final String TEST_DIRECTORY_KEY = "test.build.data";
+/*
protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
+*/
+ protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
+ protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
+ protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
+
protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
private boolean localfs = false;
@@ -159,9 +165,8 @@ public abstract class HBaseTestCase exte
Path rootdir = filesystem.makeQualified(
new Path(conf.get(HConstants.HBASE_DIR)));
filesystem.mkdirs(rootdir);
-
- return HRegion.createHRegion(new HRegionInfo(desc, startKey, endKey),
- rootdir, conf);
+ HRegionInfo hri = new HRegionInfo(desc.getName(), startKey, endKey);
+ return HRegion.createHRegion(hri, rootdir, conf, desc);
}
protected HRegion openClosedRegion(final HRegion closedRegion)
@@ -653,9 +658,10 @@ public abstract class HBaseTestCase exte
}
protected void createRootAndMetaRegions() throws IOException {
- root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, conf);
+ root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir,
+ conf, HTableDescriptor.ROOT_TABLEDESC);
meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir,
- conf);
+ conf, HTableDescriptor.META_TABLEDESC);
HRegion.addRegionToMETA(root, meta);
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Tue Jun 14 05:40:32 2011
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.migration.HRegionInfo090x;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -831,7 +832,7 @@ public class HBaseTestingUtility {
int count = 0;
for (int i = 0; i < startKeys.length; i++) {
int j = (i + 1) % startKeys.length;
- HRegionInfo hri = new HRegionInfo(table.getTableDescriptor(),
+ HRegionInfo hri = new HRegionInfo(table.getTableName(),
startKeys[i], startKeys[j]);
Put put = new Put(hri.getRegionName());
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
@@ -859,6 +860,65 @@ public class HBaseTestingUtility {
return count;
}
+ public int createMultiRegionsWithLegacyHRI(final Configuration c,
+ final HTableDescriptor htd,
+ final byte[] columnFamily, byte [][] startKeys)
+ throws IOException {
+ Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
+ HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
+ if(!htd.hasFamily(columnFamily)) {
+ HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
+ htd.addFamily(hcd);
+ }
+ List<HRegionInfo090x> newRegions
+ = new ArrayList<HRegionInfo090x>(startKeys.length);
+ int count = 0;
+ for (int i = 0; i < startKeys.length; i++) {
+ int j = (i + 1) % startKeys.length;
+ HRegionInfo090x hri = new HRegionInfo090x(htd,
+ startKeys[i], startKeys[j]);
+ Put put = new Put(hri.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(hri));
+ meta.put(put);
+ LOG.info("createMultiRegions: PUT inserted " + hri.toString());
+
+ newRegions.add(hri);
+ count++;
+ }
+ return count;
+
+ }
+
+ public int createMultiRegionsWithNewHRI(final Configuration c, final HTableDescriptor htd,
+ final byte[] columnFamily, byte [][] startKeys)
+ throws IOException {
+ Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
+ HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
+ if(!htd.hasFamily(columnFamily)) {
+ HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
+ htd.addFamily(hcd);
+ }
+ List<HRegionInfo> newRegions
+ = new ArrayList<HRegionInfo>(startKeys.length);
+ int count = 0;
+ for (int i = 0; i < startKeys.length; i++) {
+ int j = (i + 1) % startKeys.length;
+ HRegionInfo hri = new HRegionInfo(htd.getName(),
+ startKeys[i], startKeys[j]);
+ Put put = new Put(hri.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(hri));
+ meta.put(put);
+ LOG.info("createMultiRegions: PUT inserted " + hri.toString());
+
+ newRegions.add(hri);
+ count++;
+ }
+ return count;
+
+ }
+
/**
* Create rows in META for regions of the specified table with the specified
* start keys. The first startKey should be a 0 length byte array if you
@@ -879,7 +939,8 @@ public class HBaseTestingUtility {
int count = 0;
for (int i = 0; i < startKeys.length; i++) {
int j = (i + 1) % startKeys.length;
- HRegionInfo hri = new HRegionInfo(htd, startKeys[i], startKeys[j]);
+ HRegionInfo hri = new HRegionInfo(htd.getName(), startKeys[i],
+ startKeys[j]);
Put put = new Put(hri.getRegionName());
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(hri));
@@ -923,8 +984,7 @@ public class HBaseTestingUtility {
for (Result result : s) {
HRegionInfo info = Writables.getHRegionInfo(
result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
- HTableDescriptor desc = info.getTableDesc();
- if (Bytes.compareTo(desc.getName(), tableName) == 0) {
+ if (Bytes.compareTo(info.getTableName(), tableName) == 0) {
LOG.info("getMetaTableRows: row -> " +
Bytes.toStringBinary(result.getRow()));
rows.add(result.getRow());
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestCompare.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestCompare.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestCompare.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestCompare.java Tue Jun 14 05:40:32 2011
@@ -31,25 +31,25 @@ public class TestCompare extends TestCas
* Sort of HRegionInfo.
*/
public void testHRegionInfo() {
- HRegionInfo a = new HRegionInfo(new HTableDescriptor("a"), null, null);
- HRegionInfo b = new HRegionInfo(new HTableDescriptor("b"), null, null);
+ HRegionInfo a = new HRegionInfo(Bytes.toBytes("a"), null, null);
+ HRegionInfo b = new HRegionInfo(Bytes.toBytes("b"), null, null);
assertTrue(a.compareTo(b) != 0);
HTableDescriptor t = new HTableDescriptor("t");
byte [] midway = Bytes.toBytes("midway");
- a = new HRegionInfo(t, null, midway);
- b = new HRegionInfo(t, midway, null);
+ a = new HRegionInfo(t.getName(), null, midway);
+ b = new HRegionInfo(t.getName(), midway, null);
assertTrue(a.compareTo(b) < 0);
assertTrue(b.compareTo(a) > 0);
assertEquals(a, a);
assertTrue(a.compareTo(a) == 0);
- a = new HRegionInfo(t, Bytes.toBytes("a"), Bytes.toBytes("d"));
- b = new HRegionInfo(t, Bytes.toBytes("e"), Bytes.toBytes("g"));
+ a = new HRegionInfo(t.getName(), Bytes.toBytes("a"), Bytes.toBytes("d"));
+ b = new HRegionInfo(t.getName(), Bytes.toBytes("e"), Bytes.toBytes("g"));
assertTrue(a.compareTo(b) < 0);
- a = new HRegionInfo(t, Bytes.toBytes("aaaa"), Bytes.toBytes("dddd"));
- b = new HRegionInfo(t, Bytes.toBytes("e"), Bytes.toBytes("g"));
+ a = new HRegionInfo(t.getName(), Bytes.toBytes("aaaa"), Bytes.toBytes("dddd"));
+ b = new HRegionInfo(t.getName(), Bytes.toBytes("e"), Bytes.toBytes("g"));
assertTrue(a.compareTo(b) < 0);
- a = new HRegionInfo(t, Bytes.toBytes("aaaa"), Bytes.toBytes("dddd"));
- b = new HRegionInfo(t, Bytes.toBytes("aaaa"), Bytes.toBytes("eeee"));
+ a = new HRegionInfo(t.getName(), Bytes.toBytes("aaaa"), Bytes.toBytes("dddd"));
+ b = new HRegionInfo(t.getName(), Bytes.toBytes("aaaa"), Bytes.toBytes("eeee"));
assertTrue(a.compareTo(b) < 0);
}
}
\ No newline at end of file
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestScanMultipleVersions.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestScanMultipleVersions.java?rev=1135385&r1=1135384&r2=1135385&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestScanMultipleVersions.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestScanMultipleVersions.java Tue Jun 14 05:40:32 2011
@@ -57,10 +57,10 @@ public class TestScanMultipleVersions ex
this.desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
// Region 0 will contain the key range [,row_0500)
- INFOS[0] = new HRegionInfo(this.desc, HConstants.EMPTY_START_ROW,
+ INFOS[0] = new HRegionInfo(desc.getName(), HConstants.EMPTY_START_ROW,
Bytes.toBytes("row_0500"));
// Region 1 will contain the key range [row_0500,)
- INFOS[1] = new HRegionInfo(this.desc, Bytes.toBytes("row_0500"),
+ INFOS[1] = new HRegionInfo(desc.getName(), Bytes.toBytes("row_0500"),
HConstants.EMPTY_END_ROW);
// Create root and meta regions
@@ -68,7 +68,8 @@ public class TestScanMultipleVersions ex
// Create the regions
for (int i = 0; i < REGIONS.length; i++) {
REGIONS[i] =
- HRegion.createHRegion(this.INFOS[i], this.testDir, this.conf);
+ HRegion.createHRegion(this.INFOS[i], this.testDir, this.conf,
+ this.desc);
// Insert data
for (int j = 0; j < TIMESTAMPS.length; j++) {
Put put = new Put(ROWS[i], TIMESTAMPS[j], null);
@@ -194,4 +195,4 @@ public class TestScanMultipleVersions ex
s.close();
}
}
-}
\ No newline at end of file
+}