You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/10/27 23:08:34 UTC
svn commit: r1190030 - in /hbase/branches/0.92: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/catalog/
src/main/java/org/apache/hadoop/hbase/master/
src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/java/org...
Author: stack
Date: Thu Oct 27 21:08:33 2011
New Revision: 1190030
URL: http://svn.apache.org/viewvc?rev=1190030&view=rev
Log:
HBASE-4388 Second start after migration from 90 to trunk crashes
Added:
hbase/branches/0.92/src/test/data/
hbase/branches/0.92/src/test/data/hbase-4388-root.dir.tgz (with props)
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/migration/
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java
Removed:
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
Modified:
hbase/branches/0.92/CHANGES.txt
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/Writables.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Thu Oct 27 21:08:33 2011
@@ -378,6 +378,7 @@ Release 0.92.0 - Unreleased
wrong place (nkeywal)
HBASE-4673 NPE in HFileReaderV2.close during major compaction when
hfile.block.cache.size is set to 0 (Lars H)
+ HBASE-4388 Second start after migration from 90 to trunk crashes
TESTS
HBASE-4492 TestRollingRestart fails intermittently
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu Oct 27 21:08:33 2011
@@ -220,11 +220,6 @@ public final class HConstants {
// be the first to be reassigned if the server(s) they are being served by
// should go down.
-
- //
- // New stuff. Making a slow transition.
- //
-
/** The root table's name.*/
public static final byte [] ROOT_TABLE_NAME = Bytes.toBytes("-ROOT-");
@@ -255,6 +250,22 @@ public final class HConstants {
/** The upper-half split region column qualifier */
public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
+ /**
+ * The meta table version column qualifier.
+ * We keep current version of the meta table in this column in <code>-ROOT-</code>
+ * table: i.e. in the 'info:v' column.
+ */
+ public static final byte [] META_VERSION_QUALIFIER = Bytes.toBytes("v");
+
+ /**
+ * The current version of the meta table.
+ * Before this the meta had HTableDescriptor serialized into the HRegionInfo;
+ * i.e. pre-hbase 0.92. There was no META_VERSION column in the root table
+ * in this case. The presence of a version and its value being zero indicates
+ * meta is up-to-date.
+ */
+ public static final short META_VERSION = 0;
+
// Other constants
/**
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Thu Oct 27 21:08:33 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase;
import java.io.DataInput;
import java.io.DataOutput;
+import java.io.EOFException;
import java.io.IOException;
import java.util.Arrays;
@@ -43,8 +44,11 @@ import org.apache.hadoop.io.WritableComp
* Contains HRegion id, start and end keys, a reference to this
* HRegions' table descriptor, etc.
*/
-public class HRegionInfo extends VersionedWritable implements WritableComparable<HRegionInfo>{
- private static final byte VERSION = 0;
+public class HRegionInfo extends VersionedWritable
+implements WritableComparable<HRegionInfo> {
+ // VERSION == 0 when HRegionInfo had an HTableDescriptor inside it.
+ public static final byte VERSION_PRE_092 = 0;
+ public static final byte VERSION = 1;
private static final Log LOG = LogFactory.getLog(HRegionInfo.class);
/**
@@ -159,7 +163,6 @@ public class HRegionInfo extends Version
// Current TableName
private byte[] tableName = null;
- private String tableNameAsString = null;
private void setHashCode() {
int result = Arrays.hashCode(this.regionName);
@@ -710,16 +713,41 @@ public class HRegionInfo extends Version
@Override
public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- this.endKey = Bytes.readByteArray(in);
- this.offLine = in.readBoolean();
- this.regionId = in.readLong();
- this.regionName = Bytes.readByteArray(in);
- this.regionNameStr = Bytes.toStringBinary(this.regionName);
- this.split = in.readBoolean();
- this.startKey = Bytes.readByteArray(in);
- this.tableName = Bytes.readByteArray(in);
- this.hashCode = in.readInt();
+ // Read the single version byte. We don't ask the super class do it
+ // because freaks out if its not the current classes' version. This method
+ // can deserialize version 0 and version 1 of HRI.
+ byte version = in.readByte();
+ if (version == 0) {
+ // This is the old HRI that carried an HTD. Migrate it. The below
+ // was copied from the old 0.90 HRI readFields.
+ this.endKey = Bytes.readByteArray(in);
+ this.offLine = in.readBoolean();
+ this.regionId = in.readLong();
+ this.regionName = Bytes.readByteArray(in);
+ this.regionNameStr = Bytes.toStringBinary(this.regionName);
+ this.split = in.readBoolean();
+ this.startKey = Bytes.readByteArray(in);
+ try {
+ HTableDescriptor htd = new HTableDescriptor();
+ htd.readFields(in);
+ this.tableName = htd.getName();
+ } catch(EOFException eofe) {
+ throw new IOException("HTD not found in input buffer", eofe);
+ }
+ this.hashCode = in.readInt();
+ } else if (getVersion() == VERSION) {
+ this.endKey = Bytes.readByteArray(in);
+ this.offLine = in.readBoolean();
+ this.regionId = in.readLong();
+ this.regionName = Bytes.readByteArray(in);
+ this.regionNameStr = Bytes.toStringBinary(this.regionName);
+ this.split = in.readBoolean();
+ this.startKey = Bytes.readByteArray(in);
+ this.tableName = Bytes.readByteArray(in);
+ this.hashCode = in.readInt();
+ } else {
+ throw new IOException("Non-migratable/unknown version=" + getVersion());
+ }
}
//
@@ -762,4 +790,4 @@ public class HRegionInfo extends Version
return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()?
KeyValue.META_COMPARATOR: KeyValue.COMPARATOR;
}
-}
+}
\ No newline at end of file
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java Thu Oct 27 21:08:33 2011
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.util.Writ
/**
* Writes region and assignment information to <code>.META.</code>.
+ * TODO: Put MetaReader and MetaEditor together; doesn't make sense having
+ * them distinct.
*/
public class MetaEditor {
// TODO: Strip CatalogTracker from this class. Its all over and in the end
@@ -77,14 +79,12 @@ public class MetaEditor {
/**
* Put the passed <code>p</code> to a catalog table.
* @param ct CatalogTracker on whose back we will ride the edit.
- * @param regionName Name of the catalog table to put too.
* @param p Put to add
* @throws IOException
*/
- static void putToCatalogTable(final CatalogTracker ct,
- final byte [] regionName, final Put p)
+ static void putToCatalogTable(final CatalogTracker ct, final Put p)
throws IOException {
- HTable t = MetaReader.getCatalogHTable(ct, regionName);
+ HTable t = MetaReader.getCatalogHTable(ct, p.getRow());
put(t, p);
}
@@ -254,10 +254,9 @@ public class MetaEditor {
private static void updateLocation(final CatalogTracker catalogTracker,
HRegionInfo regionInfo, ServerName sn)
throws IOException {
- final byte [] regionName = regionInfo.getRegionName();
Put put = new Put(regionInfo.getRegionName());
addLocation(put, sn);
- putToCatalogTable(catalogTracker, regionName, put);
+ putToCatalogTable(catalogTracker, put);
LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
" with server=" + sn);
}
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java Thu Oct 27 21:08:33 2011
@@ -18,8 +18,9 @@
package org.apache.hadoop.hbase.catalog;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,36 +44,19 @@ import org.apache.hadoop.hbase.util.Writ
public class MetaMigrationRemovingHTD {
private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class);
- /** The metaupdated column qualifier */
- public static final byte [] META_MIGRATION_QUALIFIER =
- Bytes.toBytes("metamigrated");
-
/**
* Update legacy META rows, removing HTD from HRI.
* @param masterServices
* @return List of table descriptors.
* @throws IOException
*/
- public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
+ public static Set<HTableDescriptor> updateMetaWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
- final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
- if (hrfm == null) return true;
- htds.add(hrfm.getTableDesc());
- masterServices.getMasterFileSystem()
- .createTableDescriptor(hrfm.getTableDesc());
- updateHRI(masterServices.getCatalogTracker(), false, hrfm);
- return true;
- }
- };
+ MigratingVisitor v = new MigratingVisitor(masterServices);
MetaReader.fullScan(masterServices.getCatalogTracker(), v);
- MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker(), true);
- return htds;
+ updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker());
+ return v.htds;
}
/**
@@ -81,116 +65,143 @@ public class MetaMigrationRemovingHTD {
* @return List of table descriptors
* @throws IOException
*/
- public static List<HTableDescriptor> updateRootWithNewRegionInfo(
+ static Set<HTableDescriptor> updateRootWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
- final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
- if (hrfm == null) return true;
- htds.add(hrfm.getTableDesc());
- masterServices.getMasterFileSystem().createTableDescriptor(
- hrfm.getTableDesc());
- updateHRI(masterServices.getCatalogTracker(), true, hrfm);
+ MigratingVisitor v = new MigratingVisitor(masterServices);
+ MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true);
+ return v.htds;
+ }
+
+ /**
+ * Meta visitor that migrates the info:regioninfo as it visits.
+ */
+ static class MigratingVisitor implements Visitor {
+ private final MasterServices services;
+ final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
+
+ MigratingVisitor(final MasterServices services) {
+ this.services = services;
+ }
+
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ // Check info:regioninfo, info:splitA, and info:splitB. Make sure all
+ // have migrated HRegionInfos... that there are no leftover 090 version
+ // HRegionInfos.
+ byte [] hriBytes = getBytes(r, HConstants.REGIONINFO_QUALIFIER);
+ // Presumes that an edit updating all three cells either succeeds or
+ // doesn't -- that we don't have case of info:regioninfo migrated but not
+ // info:splitA.
+ if (isMigrated(hriBytes)) return true;
+ // OK. Need to migrate this row in meta.
+ HRegionInfo090x hri090 = getHRegionInfo090x(hriBytes);
+ HTableDescriptor htd = hri090.getTableDesc();
+ if (htd == null) {
+ LOG.warn("A 090 HRI has null HTD? Continuing; " + hri090.toString());
return true;
}
- };
- MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true);
- return htds;
+ if (!this.htds.contains(htd)) {
+ // If first time we are adding a table, then write it out to fs.
+ // Presumes that first region in table has THE table's schema which
+ // might not be too bad of a presumption since it'll be first region
+ // 'altered'
+ this.services.getMasterFileSystem().createTableDescriptor(htd);
+ this.htds.add(htd);
+ }
+ // This will 'migrate' the hregioninfo from 090 version to 092.
+ HRegionInfo hri = new HRegionInfo(hri090);
+ // Now make a put to write back to meta.
+ Put p = new Put(hri.getRegionName());
+ p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(hri));
+ // Now check info:splitA and info:splitB if present. Migrate these too.
+ checkSplit(r, p, HConstants.SPLITA_QUALIFIER);
+ checkSplit(r, p, HConstants.SPLITB_QUALIFIER);
+ // Below we fake out putToCatalogTable
+ MetaEditor.putToCatalogTable(this.services.getCatalogTracker(), p);
+ LOG.info("Migrated " + Bytes.toString(p.getRow()));
+ return true;
+ }
+ }
+
+ static void checkSplit(final Result r, final Put p, final byte [] which)
+ throws IOException {
+ byte [] hriSplitBytes = getBytes(r, which);
+ if (!isMigrated(hriSplitBytes)) {
+ // This will convert the HRI from 090 to 092 HRI.
+ HRegionInfo hri = Writables.getHRegionInfo(hriSplitBytes);
+ p.add(HConstants.CATALOG_FAMILY, which, Writables.getBytes(hri));
+ }
}
/**
- * Migrate root and meta to newer version. This updates the META and ROOT
- * and removes the HTD from HRI.
- * @param masterServices
- * @throws IOException
+ * @param r Result to dig in.
+ * @param qualifier Qualifier to look at in the passed <code>r</code>.
+ * @return Bytes for an HRegionInfo or null if no bytes or empty bytes found.
*/
- public static void migrateRootAndMeta(final MasterServices masterServices)
- throws IOException {
- updateRootWithNewRegionInfo(masterServices);
- updateMetaWithNewRegionInfo(masterServices);
+ static byte [] getBytes(final Result r, final byte [] qualifier) {
+ byte [] hriBytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
+ if (hriBytes == null || hriBytes.length <= 0) return null;
+ return hriBytes;
}
/**
- * Update the metamigrated flag in -ROOT-.
- * @param catalogTracker
- * @param metaUpdated
+ * @param r Result to look in.
+ * @param qualifier What to look at in the passed result.
+ * @return Either a 090 vintage HRegionInfo OR null if no HRegionInfo or
+ * the HRegionInfo is up to date and not in need of migration.
* @throws IOException
*/
- public static void updateRootWithMetaMigrationStatus(
- CatalogTracker catalogTracker, boolean metaUpdated)
+ static HRegionInfo090x get090HRI(final Result r, final byte [] qualifier)
throws IOException {
- Put p = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
- MetaMigrationRemovingHTD.addMetaUpdateStatus(p, metaUpdated);
- MetaEditor.putToRootTable(catalogTracker, p);
- LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
+ byte [] hriBytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
+ if (hriBytes == null || hriBytes.length <= 0) return null;
+ if (isMigrated(hriBytes)) return null;
+ return getHRegionInfo090x(hriBytes);
}
- static void updateHRI(final CatalogTracker ct, final boolean rootTable,
- final HRegionInfo090x hRegionInfo090x)
- throws IOException {
- HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
- Put p = new Put(regionInfo.getRegionName());
- p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
- Writables.getBytes(regionInfo));
- if (rootTable) {
- MetaEditor.putToRootTable(ct, p);
- } else {
- MetaEditor.putToMetaTable(ct, p);
- }
- LOG.info("Updated region " + regionInfo + " to " +
- (rootTable? "-ROOT-": ".META."));
+ static boolean isMigrated(final byte [] hriBytes) {
+ if (hriBytes == null || hriBytes.length <= 0) return true;
+ // Else, what version this HRegionInfo instance is at. The first byte
+ // is the version byte in a serialized HRegionInfo. If its same as our
+ // current HRI, then nothing to do.
+ if (hriBytes[0] == HRegionInfo.VERSION) return true;
+ if (hriBytes[0] == HRegionInfo.VERSION_PRE_092) return false;
+ // Unknown version. Return true that its 'migrated' but log warning.
+ // Should 'never' happen.
+ assert false: "Unexpected version; bytes=" + Bytes.toStringBinary(hriBytes);
+ return true;
}
/**
- * @deprecated Going away in 0.94; used for migrating to 0.92 only.
+ * Migrate root and meta to newer version. This updates the META and ROOT
+ * and removes the HTD from HRI.
+ * @param masterServices
+ * @throws IOException
*/
- public static HRegionInfo090x getHRegionInfoForMigration(
- Result data) throws IOException {
- HRegionInfo090x info = null;
- byte [] bytes =
- data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
- if (bytes == null) return null;
- try {
- info = Writables.getHRegionInfoForMigration(bytes);
- } catch(IOException ioe) {
- if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
- return null;
- } else {
- throw ioe;
- }
- }
- LOG.info("Current INFO from scan results = " + info);
- return info;
+ public static void migrateRootAndMeta(final MasterServices masterServices)
+ throws IOException {
+ updateRootWithNewRegionInfo(masterServices);
+ updateMetaWithNewRegionInfo(masterServices);
}
- public static List<HRegionInfo090x> fullScanMetaAndPrintHRIM(
- CatalogTracker catalogTracker)
+ /**
+ * Update the version flag in -ROOT-.
+ * @param catalogTracker
+ * @throws IOException
+ */
+ public static void updateRootWithMetaMigrationStatus(final CatalogTracker catalogTracker)
throws IOException {
- final List<HRegionInfo090x> regions =
- new ArrayList<HRegionInfo090x>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r);
- HRegionInfo090x hrim = getHRegionInfoForMigration(r);
- LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim);
- regions.add(hrim);
- return true;
- }
- };
- MetaReader.fullScan(catalogTracker, v);
- return regions;
+ Put p = new Put(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
+ MetaEditor.putToRootTable(catalogTracker, setMetaVersion(p));
+ LOG.info("Updated -ROOT- meta version=" + HConstants.META_VERSION);
}
- static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) {
- p.add(HConstants.CATALOG_FAMILY,
- MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
- Bytes.toBytes(metaUpdated));
+ static Put setMetaVersion(final Put p) {
+ p.add(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER,
+ Bytes.toBytes(HConstants.META_VERSION));
return p;
}
@@ -201,22 +212,27 @@ public class MetaMigrationRemovingHTD {
// Public because used in tests
public static boolean isMetaHRIUpdated(final MasterServices services)
throws IOException {
- boolean metaUpdated = false;
- List<Result> results =
- MetaReader.fullScanOfRoot(services.getCatalogTracker());
+ List<Result> results = MetaReader.fullScanOfRoot(services.getCatalogTracker());
if (results == null || results.isEmpty()) {
- LOG.info("metaUpdated = NULL.");
- return metaUpdated;
+ LOG.info("Not migrated");
+ return false;
}
- // Presume only the one result.
+ // Presume only the one result because we only support on meta region.
Result r = results.get(0);
- byte [] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY,
- MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER);
- if (metaMigrated != null && metaMigrated.length > 0) {
- metaUpdated = Bytes.toBoolean(metaMigrated);
- }
- LOG.info("Meta updated status = " + metaUpdated);
- return metaUpdated;
+ short version = getMetaVersion(r);
+ boolean migrated = version >= HConstants.META_VERSION;
+ LOG.info("Meta version=" + version + "; migrated=" + migrated);
+ return migrated;
+ }
+
+ /**
+ * @param r Result to look at
+ * @return Current meta table version or -1 if no version found.
+ */
+ static short getMetaVersion(final Result r) {
+ byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.META_VERSION_QUALIFIER);
+ return value == null || value.length <= 0? -1: Bytes.toShort(value);
}
/**
@@ -239,4 +255,21 @@ public class MetaMigrationRemovingHTD {
"Master startup aborted.");
}
}
+
+ /**
+ * Get HREgionInfoForMigration serialized from bytes.
+ * @param bytes serialized bytes
+ * @return An instance of a 090 HRI or null if we failed deserialize
+ */
+ public static HRegionInfo090x getHRegionInfo090x(final byte [] bytes) {
+ if (bytes == null || bytes.length == 0) return null;
+ HRegionInfo090x hri = null;
+ try {
+ hri = (HRegionInfo090x)Writables.getWritable(bytes, new HRegionInfo090x());
+ } catch (IOException ioe) {
+ LOG.warn("Failed deserialize as a 090 HRegionInfo); bytes=" +
+ Bytes.toStringBinary(bytes), ioe);
+ }
+ return hri;
+ }
}
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Thu Oct 27 21:08:33 2011
@@ -63,17 +63,17 @@ public class MetaReader {
}
/**
- * @param regionName
- * @return True if <code>regionName</code> is from <code>.META.</code> table.
+ * @param row
+ * @return True if <code>row</code> is row of <code>-ROOT-</code> table.
*/
- private static boolean isMetaRegion(final byte [] regionName) {
- if (regionName.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) {
+ private static boolean isRootTableRow(final byte [] row) {
+ if (row.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) {
// Can't be meta table region.
return false;
}
- // Compare the prefix of regionName. If it matches META_REGION_PREFIX prefix,
- // then this is region from .META. table.
- return Bytes.equals(regionName, 0, META_REGION_PREFIX.length,
+ // Compare the prefix of row. If it matches META_REGION_PREFIX prefix,
+ // then this is row from -ROOT_ table.
+ return Bytes.equals(row, 0, META_REGION_PREFIX.length,
META_REGION_PREFIX, 0, META_REGION_PREFIX.length);
}
@@ -199,14 +199,14 @@ public class MetaReader {
/**
* Callers should call close on the returned {@link HTable} instance.
* @param catalogTracker
- * @param regionName
+ * @param row Row we are putting
* @return
* @throws IOException
*/
static HTable getCatalogHTable(final CatalogTracker catalogTracker,
- final byte [] regionName)
+ final byte [] row)
throws IOException {
- return isMetaRegion(regionName)?
+ return isRootTableRow(row)?
getRootHTable(catalogTracker):
getMetaHTable(catalogTracker);
}
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Thu Oct 27 21:08:33 2011
@@ -410,17 +410,6 @@ public class MasterFileSystem {
}
/**
- * Get table info path for a table.
- * @param tableName
- * @return Table info path
- */
- private Path getTableInfoPath(byte[] tableName) {
- Path tablePath = new Path(this.rootdir, Bytes.toString(tableName));
- Path tableInfoPath = new Path(tablePath, HConstants.TABLEINFO_NAME);
- return tableInfoPath;
- }
-
- /**
* Create new HTableDescriptor in HDFS.
*
* @param htableDescriptor
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Oct 27 21:08:33 2011
@@ -3074,16 +3074,15 @@ public class HRegion implements HeapSize
byte[] row = r.getRegionName();
Integer lid = meta.obtainRowLock(row);
try {
- final List<KeyValue> edits = new ArrayList<KeyValue>(1);
+ final long now = EnvironmentEdgeManager.currentTimeMillis();
+ final List<KeyValue> edits = new ArrayList<KeyValue>(2);
edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER,
- EnvironmentEdgeManager.currentTimeMillis(),
- Writables.getBytes(r.getRegionInfo())));
+ HConstants.REGIONINFO_QUALIFIER, now,
+ Writables.getBytes(r.getRegionInfo())));
+ // Set into the root table the version of the meta table.
edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
- org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
- EnvironmentEdgeManager.currentTimeMillis(),
- Bytes.toBytes(true)));
-
+ HConstants.META_VERSION_QUALIFIER, now,
+ Bytes.toBytes(HConstants.META_VERSION)));
meta.put(HConstants.CATALOG_FAMILY, edits);
} finally {
meta.releaseRowLock(lid);
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/Writables.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/Writables.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/Writables.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/Writables.java Thu Oct 27 21:08:33 2011
@@ -216,16 +216,4 @@ public class Writables {
}
return tgt;
}
-
- /**
- * Get HREgionInfoForMigration serialized from bytes.
- * @param bytes serialized bytes
- * @return HRegionInfoForMigration
- * @throws IOException
- */
- public static HRegionInfo090x getHRegionInfoForMigration(final byte [] bytes)
- throws IOException {
- return (HRegionInfo090x)getWritable(bytes, new HRegionInfo090x());
- }
-
}
\ No newline at end of file
Added: hbase/branches/0.92/src/test/data/hbase-4388-root.dir.tgz
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/data/hbase-4388-root.dir.tgz?rev=1190030&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hbase/branches/0.92/src/test/data/hbase-4388-root.dir.tgz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1190030&r1=1190029&r2=1190030&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Thu Oct 27 21:08:33 2011
@@ -1017,97 +1017,6 @@ public class HBaseTestingUtility {
return count;
}
- public int createMultiRegionsWithLegacyHRI(final Configuration c,
- final HTableDescriptor htd,
- final byte [] family, int numRegions)
- throws IOException {
- if (numRegions < 3) throw new IOException("Must create at least 3 regions");
- byte [] startKey = Bytes.toBytes("aaaaa");
- byte [] endKey = Bytes.toBytes("zzzzz");
- byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
- byte [][] regionStartKeys = new byte[splitKeys.length+1][];
- for (int i=0;i<splitKeys.length;i++) {
- regionStartKeys[i+1] = splitKeys[i];
- }
- regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
- return createMultiRegionsWithLegacyHRI(c, htd, family, regionStartKeys);
- }
-
- public int createMultiRegionsWithLegacyHRI(final Configuration c,
- final HTableDescriptor htd,
- final byte[] columnFamily, byte [][] startKeys)
- throws IOException {
- Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
- HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
- if(!htd.hasFamily(columnFamily)) {
- HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
- htd.addFamily(hcd);
- }
- List<HRegionInfo090x> newRegions
- = new ArrayList<HRegionInfo090x>(startKeys.length);
- int count = 0;
- for (int i = 0; i < startKeys.length; i++) {
- int j = (i + 1) % startKeys.length;
- HRegionInfo090x hri = new HRegionInfo090x(htd,
- startKeys[i], startKeys[j]);
- Put put = new Put(hri.getRegionName());
- put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
- Writables.getBytes(hri));
- meta.put(put);
- LOG.info("createMultiRegions: PUT inserted " + hri.toString());
-
- newRegions.add(hri);
- count++;
- }
- return count;
-
- }
-
- public int createMultiRegionsWithNewHRI(final Configuration c,
- final HTableDescriptor htd,
- final byte [] family, int numRegions)
- throws IOException {
- if (numRegions < 3) throw new IOException("Must create at least 3 regions");
- byte [] startKey = Bytes.toBytes("aaaaa");
- byte [] endKey = Bytes.toBytes("zzzzz");
- byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
- byte [][] regionStartKeys = new byte[splitKeys.length+1][];
- for (int i=0;i<splitKeys.length;i++) {
- regionStartKeys[i+1] = splitKeys[i];
- }
- regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
- return createMultiRegionsWithNewHRI(c, htd, family, regionStartKeys);
- }
-
- public int createMultiRegionsWithNewHRI(final Configuration c, final HTableDescriptor htd,
- final byte[] columnFamily, byte [][] startKeys)
- throws IOException {
- Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
- HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
- if(!htd.hasFamily(columnFamily)) {
- HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
- htd.addFamily(hcd);
- }
- List<HRegionInfo> newRegions
- = new ArrayList<HRegionInfo>(startKeys.length);
- int count = 0;
- for (int i = 0; i < startKeys.length; i++) {
- int j = (i + 1) % startKeys.length;
- HRegionInfo hri = new HRegionInfo(htd.getName(),
- startKeys[i], startKeys[j]);
- Put put = new Put(hri.getRegionName());
- put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
- Writables.getBytes(hri));
- meta.put(put);
- LOG.info("createMultiRegions: PUT inserted " + hri.toString());
-
- newRegions.add(hri);
- count++;
- }
- return count;
-
- }
-
/**
* Create rows in META for regions of the specified table with the specified
* start keys. The first startKey should be a 0 length byte array if you
Added: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java?rev=1190030&view=auto
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java (added)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java Thu Oct 27 21:08:33 2011
@@ -0,0 +1,354 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import junit.framework.Assert;
+import junit.framework.AssertionFailedError;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.migration.HRegionInfo090x;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test migration that removes HTableDescriptor from HRegionInfo moving the
+ * meta version from no version to {@link MetaReader#META_VERSION}.
+ */
+public class TestMetaMigrationRemovingHTD {
+ static final Log LOG = LogFactory.getLog(TestMetaMigrationRemovingHTD.class);
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final static String TESTTABLE = "TestTable";
+ private final static int ROWCOUNT = 100;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // Start up our mini cluster on top of an 0.90 root.dir that has data from
+ // a 0.90 hbase run -- it has a table with 100 rows in it -- and see if
+ // we can migrate from 0.90.
+ TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniDFSCluster(1);
+ Path testdir = TEST_UTIL.getDataTestDir("TestMetaMigrationRemovingHTD");
+ // Untar our test dir.
+ File untar = untar(new File(testdir.toString()));
+ // Now copy the untar up into hdfs so when we start hbase, we'll run from it.
+ Configuration conf = TEST_UTIL.getConfiguration();
+ FsShell shell = new FsShell(conf);
+ FileSystem fs = FileSystem.get(conf);
+ // Minihbase roots itself in user home directory up in minidfs.
+ Path homedir = fs.getHomeDirectory();
+ doFsCommand(shell,
+ new String [] {"-put", untar.toURI().toString(), homedir.toString()});
+ // See whats in minihdfs.
+ doFsCommand(shell, new String [] {"-lsr", "/"});
+ TEST_UTIL.startMiniHBaseCluster(1, 1);
+ // Assert we are running against the copied-up filesystem. The copied-up
+ // rootdir should have had a table named 'TestTable' in it. Assert it
+ // present.
+ HTable t = new HTable(TEST_UTIL.getConfiguration(), TESTTABLE);
+ ResultScanner scanner = t.getScanner(new Scan());
+ int count = 0;
+ while (scanner.next() != null) {
+ count++;
+ }
+ // Assert that we find all 100 rows that are in the data we loaded. If
+ // so then we must have migrated it from 0.90 to 0.92.
+ Assert.assertEquals(ROWCOUNT, count);
+ }
+
+ private static File untar(final File testdir) throws IOException {
+ // Find the src data under src/test/data
+ final String datafile = "hbase-4388-root.dir";
+ String srcTarFile =
+ System.getProperty("project.build.testSourceDirectory", "src/test") +
+ File.separator + "data" + File.separator + datafile + ".tgz";
+ File homedir = new File(testdir.toString());
+ File tgtUntarDir = new File(homedir, datafile);
+ if (tgtUntarDir.exists()) {
+ if (!FileUtil.fullyDelete(tgtUntarDir)) {
+ throw new IOException("Failed delete of " + tgtUntarDir.toString());
+ }
+ }
+ LOG.info("Untarring " + srcTarFile + " into " + homedir.toString());
+ FileUtil.unTar(new File(srcTarFile), homedir);
+ Assert.assertTrue(tgtUntarDir.exists());
+ return tgtUntarDir;
+ }
+
+ private static void doFsCommand(final FsShell shell, final String [] args)
+ throws Exception {
+ // Run the 'put' command.
+ int errcode = shell.run(args);
+ if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testMetaUpdatedFlagInROOT() throws Exception {
+ boolean metaUpdated = MetaMigrationRemovingHTD.
+ isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster());
+ assertEquals(true, metaUpdated);
+ }
+
+ @Test
+ public void testMetaMigration() throws Exception {
+ LOG.info("Starting testMetaWithLegacyHRI");
+ final byte [] FAMILY = Bytes.toBytes("family");
+ HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
+ HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+ htd.addFamily(hcd);
+ Configuration conf = TEST_UTIL.getConfiguration();
+ createMultiRegionsWithLegacyHRI(conf, htd, FAMILY,
+ new byte[][]{
+ HConstants.EMPTY_START_ROW,
+ Bytes.toBytes("region_a"),
+ Bytes.toBytes("region_b")});
+ CatalogTracker ct =
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
+ // Erase the current version of root meta for this test.
+ undoVersionInMeta();
+ MetaReader.fullScanMetaAndPrint(ct);
+ LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
+
+ Set<HTableDescriptor> htds =
+ MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
+ TEST_UTIL.getHBaseCluster().getMaster());
+ MetaReader.fullScanMetaAndPrint(ct);
+ // Should be one entry only and it should be for the table we just added.
+ assertEquals(1, htds.size());
+ assertTrue(htds.contains(htd));
+ // Assert that the flag in ROOT is updated to reflect the correct status
+ boolean metaUpdated =
+ MetaMigrationRemovingHTD.isMetaHRIUpdated(
+ TEST_UTIL.getMiniHBaseCluster().getMaster());
+ assertEquals(true, metaUpdated);
+ }
+
+ /**
+ * This test assumes a master crash/failure during the meta migration process
+ * and attempts to continue the meta migration process when a new master takes over.
+ * When a master dies during the meta migration we will have some rows of
+ * META.CatalogFamily updated with new HRI, (i.e HRI with out HTD) and some
+ * still hanging with legacy HRI. (i.e HRI with HTD). When the backup master/ or
+ * fresh start of master attempts the migration it will encouter some rows of META
+ * already updated with new HRI and some still legacy. This test will simulate this
+ * scenario and validates that the migration process can safely skip the updated
+ * rows and migrate any pending rows at startup.
+ * @throws Exception
+ */
+ @Test
+ public void testMasterCrashDuringMetaMigration() throws Exception {
+ final byte[] FAMILY = Bytes.toBytes("family");
+ HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration");
+ HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+ htd.addFamily(hcd);
+ Configuration conf = TEST_UTIL.getConfiguration();
+ // Create 10 New regions.
+ createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10);
+ // Create 10 Legacy regions.
+ createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10);
+ CatalogTracker ct =
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
+ // Erase the current version of root meta for this test.
+ undoVersionInMeta();
+ MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct);
+ //MetaReader.fullScanMetaAndPrint(ct);
+ LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
+
+ Set<HTableDescriptor> htds =
+ MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
+ TEST_UTIL.getHBaseCluster().getMaster());
+ assertEquals(1, htds.size());
+ assertTrue(htds.contains(htd));
+ // Assert that the flag in ROOT is updated to reflect the correct status
+ boolean metaUpdated = MetaMigrationRemovingHTD.
+ isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster());
+ assertEquals(true, metaUpdated);
+ LOG.info("END testMetaWithLegacyHRI");
+ }
+
+ private void undoVersionInMeta() throws IOException {
+ Delete d = new Delete(HRegionInfo.ROOT_REGIONINFO.getRegionName());
+ // Erase the current version of root meta for this test.
+ d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER);
+ HTable rootTable =
+ new HTable(TEST_UTIL.getConfiguration(), HConstants.ROOT_TABLE_NAME);
+ try {
+ rootTable.delete(d);
+ } finally {
+ rootTable.close();
+ }
+ }
+
+ public static void assertEquals(int expected, int actual) {
+ if (expected != actual) {
+ throw new AssertionFailedError("expected:<" +
+ expected + "> but was:<" +
+ actual + ">");
+ }
+ }
+
+ public static void assertEquals(boolean expected, boolean actual) {
+ if (expected != actual) {
+ throw new AssertionFailedError("expected:<" +
+ expected + "> but was:<" +
+ actual + ">");
+ }
+ }
+
+
+ /**
+ * @param c
+ * @param htd
+ * @param family
+ * @param numRegions
+ * @return
+ * @throws IOException
+ * @deprecated Just for testing migration of meta from 0.90 to 0.92... will be
+ * removed thereafter
+ */
+ public int createMultiRegionsWithLegacyHRI(final Configuration c,
+ final HTableDescriptor htd, final byte [] family, int numRegions)
+ throws IOException {
+ if (numRegions < 3) throw new IOException("Must create at least 3 regions");
+ byte [] startKey = Bytes.toBytes("aaaaa");
+ byte [] endKey = Bytes.toBytes("zzzzz");
+ byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
+ byte [][] regionStartKeys = new byte[splitKeys.length+1][];
+ for (int i=0;i<splitKeys.length;i++) {
+ regionStartKeys[i+1] = splitKeys[i];
+ }
+ regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
+ return createMultiRegionsWithLegacyHRI(c, htd, family, regionStartKeys);
+ }
+
+ /**
+ * @param c
+ * @param htd
+ * @param columnFamily
+ * @param startKeys
+ * @return
+ * @throws IOException
+ * @deprecated Just for testing migration of meta from 0.90 to 0.92... will be
+ * removed thereafter
+ */
+ public int createMultiRegionsWithLegacyHRI(final Configuration c,
+ final HTableDescriptor htd, final byte[] columnFamily, byte [][] startKeys)
+ throws IOException {
+ Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
+ HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
+ if(!htd.hasFamily(columnFamily)) {
+ HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
+ htd.addFamily(hcd);
+ }
+ List<HRegionInfo090x> newRegions
+ = new ArrayList<HRegionInfo090x>(startKeys.length);
+ int count = 0;
+ for (int i = 0; i < startKeys.length; i++) {
+ int j = (i + 1) % startKeys.length;
+ HRegionInfo090x hri = new HRegionInfo090x(htd,
+ startKeys[i], startKeys[j]);
+ Put put = new Put(hri.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(hri));
+ meta.put(put);
+ LOG.info("createMultiRegions: PUT inserted " + hri.toString());
+
+ newRegions.add(hri);
+ count++;
+ }
+ return count;
+ }
+
+ int createMultiRegionsWithNewHRI(final Configuration c,
+ final HTableDescriptor htd, final byte [] family, int numRegions)
+ throws IOException {
+ if (numRegions < 3) throw new IOException("Must create at least 3 regions");
+ byte [] startKey = Bytes.toBytes("aaaaa");
+ byte [] endKey = Bytes.toBytes("zzzzz");
+ byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
+ byte [][] regionStartKeys = new byte[splitKeys.length+1][];
+ for (int i=0;i<splitKeys.length;i++) {
+ regionStartKeys[i+1] = splitKeys[i];
+ }
+ regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
+ return createMultiRegionsWithNewHRI(c, htd, family, regionStartKeys);
+ }
+
+ int createMultiRegionsWithNewHRI(final Configuration c, final HTableDescriptor htd,
+ final byte[] columnFamily, byte [][] startKeys)
+ throws IOException {
+ Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
+ HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
+ if(!htd.hasFamily(columnFamily)) {
+ HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
+ htd.addFamily(hcd);
+ }
+ List<HRegionInfo> newRegions
+ = new ArrayList<HRegionInfo>(startKeys.length);
+ int count = 0;
+ for (int i = 0; i < startKeys.length; i++) {
+ int j = (i + 1) % startKeys.length;
+ HRegionInfo hri = new HRegionInfo(htd.getName(),
+ startKeys[i], startKeys[j]);
+ Put put = new Put(hri.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(hri));
+ meta.put(put);
+ LOG.info("createMultiRegions: PUT inserted " + hri.toString());
+
+ newRegions.add(hri);
+ count++;
+ }
+ return count;
+ }
+}
Added: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java?rev=1190030&view=auto
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java (added)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/migration/TestMigrationFrom090To092.java Thu Oct 27 21:08:33 2011
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.migration;
+
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD;
+import org.apache.hadoop.hbase.util.Writables;
+import org.junit.Test;
+
+/**
+ * Migration tests that do not need spin up of a cluster.
+ * @deprecated Remove after we release 0.92
+ */
+public class TestMigrationFrom090To092 {
+ @Test
+ public void testMigrateHRegionInfoFromVersion0toVersion1()
+ throws IOException {
+ HTableDescriptor htd =
+ getHTableDescriptor("testMigrateHRegionInfoFromVersion0toVersion1");
+ HRegionInfo090x ninety =
+ new HRegionInfo090x(htd, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+ byte [] bytes = Writables.getBytes(ninety);
+ // Now deserialize into an HRegionInfo
+ HRegionInfo hri = Writables.getHRegionInfo(bytes);
+ Assert.assertEquals(hri.getTableNameAsString(),
+ ninety.getTableDesc().getNameAsString());
+ Assert.assertEquals(HRegionInfo.VERSION, hri.getVersion());
+ }
+
+ private HTableDescriptor getHTableDescriptor(final String name) {
+ HTableDescriptor htd = new HTableDescriptor(name);
+ htd.addFamily(new HColumnDescriptor("family"));
+ return htd;
+ }
+}
\ No newline at end of file