You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/05/20 00:46:50 UTC
svn commit: r658016 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/master/
src/java/org/apache/hadoop/hbase/regionserver/
src/java/org/apache/hadoop/hbase/util/ s...
Author: stack
Date: Mon May 19 15:46:50 2008
New Revision: 658016
URL: http://svn.apache.org/viewvc?rev=658016&view=rev
Log:
HBASE-623 migration script for hbase-82
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestClassMigration.java
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/build.xml
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Bytes.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestSerialization.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Mon May 19 15:46:50 2008
@@ -15,6 +15,7 @@
HBASE-624 Master will shut down if number of active region servers is zero
even if shutdown was not requested
HBASE-629 Split reports incorrect elapsed time
+ HBASE-623 Migration script for hbase-82
IMPROVEMENTS
HBASE-559 MR example job to count table rows
Modified: hadoop/hbase/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/build.xml?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/build.xml (original)
+++ hadoop/hbase/trunk/build.xml Mon May 19 15:46:50 2008
@@ -30,6 +30,7 @@
<property name="src.dir" location="${basedir}/src/java"/>
<property name="src.test" location="${basedir}/src/test"/>
+ <property name="src.testdata" location="${basedir}/src/testdata"/>
<property name="src.examples" location="${basedir}/src/examples"/>
<property name="src.webapps" location="${basedir}/src/webapps"/>
@@ -379,6 +380,7 @@
<sysproperty key="test.build.data" value="${build.test}/data"/>
<sysproperty key="build.test" value="${build.test}"/>
+ <sysproperty key="src.testdata" value="${src.testdata}"/>
<sysproperty key="contrib.name" value="${name}"/>
<sysproperty key="user.dir" value="${build.test}/data"/>
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java Mon May 19 15:46:50 2008
@@ -335,6 +335,9 @@
Text t = new Text();
t.readFields(in);
this.name = t.getBytes();
+ if (HStoreKey.getFamilyDelimiterIndex(this.name) > 0) {
+ this.name = stripColon(this.name);
+ }
} else {
this.name = Bytes.readByteArray(in);
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Mon May 19 15:46:50 2008
@@ -202,4 +202,8 @@
* Unlimited time-to-live.
*/
static final int FOREVER = -1;
-}
+
+ public static final String HBASE_CLIENT_RETRIES_NUMBER_KEY =
+ "hbase.client.retries.number";
+ public static final int DEFAULT_CLIENT_RETRIES = 5;
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Mon May 19 15:46:50 2008
@@ -157,19 +157,16 @@
this.masterChecked = false;
this.servers = new ConcurrentHashMap<String, HRegionInterface>();
}
-
+
/** {@inheritDoc} */
public HMasterInterface getMaster() throws MasterNotRunningException {
+ HServerAddress masterLocation = null;
synchronized (this.masterLock) {
- for (int tries = 0;
- !this.closed &&
- !this.masterChecked && this.master == null &&
- tries < numRetries;
- tries++) {
-
- HServerAddress masterLocation = new HServerAddress(this.conf.get(
- MASTER_ADDRESS, DEFAULT_MASTER_ADDRESS));
-
+ for (int tries = 0; !this.closed &&
+ !this.masterChecked && this.master == null && tries < numRetries;
+ tries++) {
+ String m = this.conf.get(MASTER_ADDRESS, DEFAULT_MASTER_ADDRESS);
+ masterLocation = new HServerAddress(m);
try {
HMasterInterface tryMaster = (HMasterInterface)HbaseRPC.getProxy(
HMasterInterface.class, HMasterInterface.versionID,
@@ -181,12 +178,12 @@
}
} catch (IOException e) {
- if(tries == numRetries - 1) {
+ if (tries == numRetries - 1) {
// This was our last chance - don't bother sleeping
break;
}
LOG.info("Attempt " + tries + " of " + this.numRetries +
- " failed with <" + e + ">. Retrying after sleep of " + this.pause);
+ " failed with <" + e + ">. Retrying after sleep of " + this.pause);
}
// We either cannot connect to master or it is not running. Sleep & retry
@@ -200,7 +197,8 @@
this.masterChecked = true;
}
if (this.master == null) {
- throw new MasterNotRunningException();
+ throw new MasterNotRunningException(masterLocation == null? "":
+ masterLocation.toString());
}
return this.master;
}
@@ -323,12 +321,11 @@
// This block guards against two threads trying to find the root
// region at the same time. One will go do the find while the
// second waits. The second thread will not do find.
-
if (!useCache || rootRegionLocation == null) {
return locateRootRegion();
}
return rootRegionLocation;
- }
+ }
} else if (Bytes.equals(tableName, META_TABLE_NAME)) {
synchronized (metaRegionLock) {
// This block guards against two threads trying to load the meta
@@ -655,30 +652,29 @@
*/
private HRegionLocation locateRootRegion()
throws IOException {
-
getMaster();
-
HServerAddress rootRegionAddress = null;
-
for (int tries = 0; tries < numRetries; tries++) {
int localTimeouts = 0;
-
- // ask the master which server has the root region
+ // Ask the master which server has the root region
while (rootRegionAddress == null && localTimeouts < numRetries) {
rootRegionAddress = master.findRootRegion();
if (rootRegionAddress == null) {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sleeping. Waiting for root region.");
- }
- Thread.sleep(pause);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Wake. Retry finding root region.");
+ // Increment and then only sleep if retries left.
+ if (++localTimeouts < numRetries) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sleeping " + pause + "ms. Waiting for root "
+ + "region. Attempt " + tries + " of " + numRetries);
+ }
+ Thread.sleep(pause);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Wake. Retry finding root region.");
+ }
+ } catch (InterruptedException iex) {
+ // continue
}
- } catch (InterruptedException iex) {
- // continue
}
- localTimeouts++;
}
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java Mon May 19 15:46:50 2008
@@ -526,6 +526,23 @@
}
/**
+ * Get a scanner on the current table starting at first row.
+ * Return the specified columns.
+ *
+ * @param columns columns to scan. If column name is a column family, all
+ * columns of the specified column family are returned. Its also possible
+ * to pass a regex in the column qualifier. A column qualifier is judged to
+ * be a regex if it contains at least one of the following characters:
+ * <code>\+|^&*$[]]}{)(</code>.
+ * @return scanner
+ * @throws IOException
+ */
+ public Scanner getScanner(final Text [] columns)
+ throws IOException {
+ return getScanner(Bytes.toByteArrays(columns), HConstants.EMPTY_START_ROW);
+ }
+
+ /**
* Get a scanner on the current table starting at the specified row.
* Return the specified columns.
*
@@ -542,7 +559,25 @@
throws IOException {
return getScanner(Bytes.toByteArrays(columns), startRow.getBytes());
}
-
+
+ /**
+ * Get a scanner on the current table starting at first row.
+ * Return the specified columns.
+ *
+ * @param columns columns to scan. If column name is a column family, all
+ * columns of the specified column family are returned. Its also possible
+ * to pass a regex in the column qualifier. A column qualifier is judged to
+ * be a regex if it contains at least one of the following characters:
+ * <code>\+|^&*$[]]}{)(</code>.
+ * @return scanner
+ * @throws IOException
+ */
+ public Scanner getScanner(final byte[][] columns)
+ throws IOException {
+ return getScanner(columns, HConstants.EMPTY_START_ROW,
+ HConstants.LATEST_TIMESTAMP, null);
+ }
+
/**
* Get a scanner on the current table starting at the specified row.
* Return the specified columns.
@@ -875,7 +910,7 @@
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating scanner over " + Bytes.toString(tableName) +
- " starting at key '" + startRow + "'");
+ " starting at key '" + Bytes.toString(startRow) + "'");
}
// save off the simple parameters
this.columns = columns;
@@ -921,7 +956,8 @@
byte [] localStartKey = oldRegion == null? startRow: oldRegion.getEndKey();
if (LOG.isDebugEnabled()) {
- LOG.debug("Advancing internal scanner to startKey " + localStartKey);
+ LOG.debug("Advancing internal scanner to startKey at " +
+ Bytes.toString(localStartKey));
}
try {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java Mon May 19 15:46:50 2008
@@ -277,7 +277,7 @@
result = true;
} else if (LOG.isDebugEnabled()) {
// If debug, note we checked and current state of daughters.
- LOG.debug("Checked " + parent.getRegionName() +
+ LOG.debug("Checked " + parent.getRegionNameAsString() +
" for references: splitA: " + hasReferencesA + ", splitB: "+
hasReferencesB);
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon May 19 15:46:50 2008
@@ -943,7 +943,7 @@
}
for(HRegion region: regionsToClose) {
if (LOG.isDebugEnabled()) {
- LOG.debug("closing region " + region.getRegionName());
+ LOG.debug("closing region " + Bytes.toString(region.getRegionName()));
}
try {
region.close();
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Mon May 19 15:46:50 2008
@@ -384,7 +384,7 @@
HStoreFile curfile = null;
HStoreFile.Reference reference = null;
if (isReference) {
- reference = readSplitInfo(p, fs);
+ reference = HStoreFile.readSplitInfo(p, fs);
}
curfile = new HStoreFile(conf, fs, basedir, info.getEncodedName(),
family.getName(), fid, reference);
@@ -1773,21 +1773,6 @@
return this.storeNameStr;
}
- /*
- * @see writeSplitInfo(Path p, HStoreFile hsf, FileSystem fs)
- */
- static HStoreFile.Reference readSplitInfo(final Path p, final FileSystem fs)
- throws IOException {
- FSDataInputStream in = fs.open(p);
- try {
- HStoreFile.Reference r = new HStoreFile.Reference();
- r.readFields(in);
- return r;
- } finally {
- in.close();
- }
- }
-
/**
* @param p Path to check.
* @return True if the path has format of a HStoreFile reference.
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java Mon May 19 15:46:50 2008
@@ -281,7 +281,22 @@
out.close();
}
}
-
+
+ /**
+ * @see #writeSplitInfo(FileSystem fs)
+ */
+ static HStoreFile.Reference readSplitInfo(final Path p, final FileSystem fs)
+ throws IOException {
+ FSDataInputStream in = fs.open(p);
+ try {
+ HStoreFile.Reference r = new HStoreFile.Reference();
+ r.readFields(in);
+ return r;
+ } finally {
+ in.close();
+ }
+ }
+
private void createOrFail(final FileSystem fs, final Path p)
throws IOException {
if (fs.exists(p)) {
@@ -585,6 +600,12 @@
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
+ // Write out the encoded region name as a String. Doing it as a String
+ // keeps a Reference's serialziation backword compatible with
+ // pre-HBASE-82 serializations. ALternative is rewriting all
+ // info files in hbase (Serialized References are written into the
+ // 'info' file that accompanies HBase Store files).
+ out.writeUTF(Integer.toString(encodedRegionName));
out.writeInt(this.encodedRegionName);
out.writeLong(fileid);
// Write true if we're doing top of the file.
@@ -594,7 +615,7 @@
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
- this.encodedRegionName = in.readInt();
+ this.encodedRegionName = Integer.parseInt(in.readUTF());
fileid = in.readLong();
boolean tmp = in.readBoolean();
// If true, set region to top.
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Bytes.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Bytes.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Bytes.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Bytes.java Mon May 19 15:46:50 2008
@@ -11,6 +11,7 @@
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
public class Bytes {
/**
@@ -40,8 +41,12 @@
*/
public static byte [] readByteArray(final DataInput in)
throws IOException {
- byte [] result = new byte[in.readInt()];
- in.readFully(result, 0, result.length);
+ int len = WritableUtils.readVInt(in);
+ if (len < 0) {
+ throw new NegativeArraySizeException(Integer.toString(len));
+ }
+ byte [] result = new byte[len];
+ in.readFully(result, 0, len);
return result;
}
@@ -52,7 +57,7 @@
*/
public static void writeByteArray(final DataOutput out, final byte [] b)
throws IOException {
- out.writeInt(b.length);
+ WritableUtils.writeVInt(out, b.length);
out.write(b, 0, b.length);
}
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestClassMigration.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestClassMigration.java?rev=658016&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestClassMigration.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestClassMigration.java Mon May 19 15:46:50 2008
@@ -0,0 +1,266 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+import junit.framework.TestCase;
+
+/**
+ * Test that individual classes can migrate themselves.
+ */
+public class TestClassMigration extends TestCase {
+
+ /**
+ * Test we can migrate a 0.1 version of HSK.
+ * @throws Exception
+ */
+ public void testMigrateHStoreKey() throws Exception {
+ long now = System.currentTimeMillis();
+ byte [] nameBytes = Bytes.toBytes(getName());
+ Text nameText = new Text(nameBytes);
+ HStoreKey01Branch hsk = new HStoreKey01Branch(nameText, nameText, now);
+ byte [] b = Writables.getBytes(hsk);
+ HStoreKey deserializedHsk =
+ (HStoreKey)Writables.getWritable(b, new HStoreKey());
+ assertEquals(deserializedHsk.getTimestamp(), hsk.getTimestamp());
+ assertTrue(Bytes.equals(nameBytes, deserializedHsk.getColumn()));
+ assertTrue(Bytes.equals(nameBytes, deserializedHsk.getRow()));
+ }
+
+ /**
+ * HBase 0.1 branch HStoreKey. Same in all regards except the utility
+ * methods have been removed.
+ * Used in test of HSK migration test.
+ */
+ private static class HStoreKey01Branch implements WritableComparable {
+ /**
+ * Colon character in UTF-8
+ */
+ public static final char COLUMN_FAMILY_DELIMITER = ':';
+
+ private Text row;
+ private Text column;
+ private long timestamp;
+
+
+ /** Default constructor used in conjunction with Writable interface */
+ public HStoreKey01Branch() {
+ this(new Text());
+ }
+
+ /**
+ * Create an HStoreKey specifying only the row
+ * The column defaults to the empty string and the time stamp defaults to
+ * Long.MAX_VALUE
+ *
+ * @param row - row key
+ */
+ public HStoreKey01Branch(Text row) {
+ this(row, Long.MAX_VALUE);
+ }
+
+ /**
+ * Create an HStoreKey specifying the row and timestamp
+ * The column name defaults to the empty string
+ *
+ * @param row row key
+ * @param timestamp timestamp value
+ */
+ public HStoreKey01Branch(Text row, long timestamp) {
+ this(row, new Text(), timestamp);
+ }
+
+ /**
+ * Create an HStoreKey specifying the row and column names
+ * The timestamp defaults to LATEST_TIMESTAMP
+ *
+ * @param row row key
+ * @param column column key
+ */
+ public HStoreKey01Branch(Text row, Text column) {
+ this(row, column, HConstants.LATEST_TIMESTAMP);
+ }
+
+ /**
+ * Create an HStoreKey specifying all the fields
+ *
+ * @param row row key
+ * @param column column key
+ * @param timestamp timestamp value
+ */
+ public HStoreKey01Branch(Text row, Text column, long timestamp) {
+ // Make copies by doing 'new Text(arg)'.
+ this.row = new Text(row);
+ this.column = new Text(column);
+ this.timestamp = timestamp;
+ }
+
+ /** @return Approximate size in bytes of this key. */
+ public long getSize() {
+ return this.row.getLength() + this.column.getLength() +
+ 8 /* There is no sizeof in java. Presume long is 8 (64bit machine)*/;
+ }
+
+ /**
+ * Constructs a new HStoreKey from another
+ *
+ * @param other the source key
+ */
+ public HStoreKey01Branch(HStoreKey01Branch other) {
+ this(other.row, other.column, other.timestamp);
+ }
+
+ /**
+ * Change the value of the row key
+ *
+ * @param newrow new row key value
+ */
+ public void setRow(Text newrow) {
+ this.row.set(newrow);
+ }
+
+ /**
+ * Change the value of the column key
+ *
+ * @param newcol new column key value
+ */
+ public void setColumn(Text newcol) {
+ this.column.set(newcol);
+ }
+
+ /**
+ * Change the value of the timestamp field
+ *
+ * @param timestamp new timestamp value
+ */
+ public void setVersion(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ /**
+ * Set the value of this HStoreKey from the supplied key
+ *
+ * @param k key value to copy
+ */
+ public void set(HStoreKey01Branch k) {
+ this.row = k.getRow();
+ this.column = k.getColumn();
+ this.timestamp = k.getTimestamp();
+ }
+
+ /** @return value of row key */
+ public Text getRow() {
+ return row;
+ }
+
+ /** @return value of column key */
+ public Text getColumn() {
+ return column;
+ }
+
+ /** @return value of timestamp */
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return row.toString() + "/" + column.toString() + "/" + timestamp;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean equals(Object obj) {
+ return compareTo(obj) == 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int hashCode() {
+ int result = this.row.hashCode();
+ result ^= this.column.hashCode();
+ result ^= this.timestamp;
+ return result;
+ }
+
+ // Comparable
+
+ public int compareTo(Object o) {
+ HStoreKey01Branch other = (HStoreKey01Branch)o;
+ int result = this.row.compareTo(other.row);
+ if (result != 0) {
+ return result;
+ }
+ result = this.column.compareTo(other.column);
+ if (result != 0) {
+ return result;
+ }
+ // The below older timestamps sorting ahead of newer timestamps looks
+ // wrong but it is intentional. This way, newer timestamps are first
+ // found when we iterate over a memcache and newer versions are the
+ // first we trip over when reading from a store file.
+ if (this.timestamp < other.timestamp) {
+ result = 1;
+ } else if (this.timestamp > other.timestamp) {
+ result = -1;
+ }
+ return result;
+ }
+
+ // Writable
+
+ /** {@inheritDoc} */
+ public void write(DataOutput out) throws IOException {
+ row.write(out);
+ column.write(out);
+ out.writeLong(timestamp);
+ }
+
+ /** {@inheritDoc} */
+ public void readFields(DataInput in) throws IOException {
+ row.readFields(in);
+ column.readFields(in);
+ timestamp = in.readLong();
+ }
+
+ /**
+ * Returns row and column bytes out of an HStoreKey.
+ * @param hsk Store key.
+ * @return byte array encoding of HStoreKey
+ * @throws UnsupportedEncodingException
+ */
+ public static byte[] getBytes(final HStoreKey hsk)
+ throws UnsupportedEncodingException {
+ StringBuilder s = new StringBuilder(hsk.getRow().toString());
+ s.append(hsk.getColumn().toString());
+ return s.toString().getBytes(HConstants.UTF8_ENCODING);
+ }
+ }
+}
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestSerialization.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestSerialization.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestSerialization.java Mon May 19 15:46:50 2008
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase;
+
import org.apache.hadoop.hbase.io.BatchOperation;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.Cell;
@@ -40,7 +41,7 @@
super.tearDown();
}
- public void testname() throws Exception {
+ public void testHMsg() throws Exception {
HMsg m = new HMsg(HMsg.MSG_REGIONSERVER_QUIESCE);
byte [] mb = Writables.getBytes(m);
HMsg deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java Mon May 19 15:46:50 2008
@@ -153,7 +153,7 @@
assertTrue(this.fs.exists(refHsf.getMapFilePath()));
assertTrue(this.fs.exists(refHsf.getInfoFilePath()));
HStoreFile.Reference otherReference =
- HStore.readSplitInfo(refHsf.getInfoFilePath(), this.fs);
+ HStoreFile.readSplitInfo(refHsf.getInfoFilePath(), this.fs);
assertEquals(reference.getEncodedRegionName(),
otherReference.getEncodedRegionName());
assertEquals(reference.getFileId(),
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java?rev=658016&r1=658015&r2=658016&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigrate.java Mon May 19 15:46:50 2008
@@ -20,57 +20,50 @@
package org.apache.hadoop.hbase.util;
+import java.io.FileNotFoundException;
import java.io.IOException;
-
-import java.util.zip.ZipInputStream;
import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Scanner;
+import org.apache.hadoop.hbase.io.RowResult;
/**
*
*/
public class TestMigrate extends HBaseTestCase {
- static final Log LOG = LogFactory.getLog(TestMigrate.class);
-
- /**
- *
- */
- public TestMigrate() {
- super();
- Logger.getRootLogger().setLevel(Level.WARN);
- Logger.getLogger(this.getClass().getPackage().getName()).
- setLevel(Level.DEBUG);
- }
+ private static final Log LOG = LogFactory.getLog(TestMigrate.class);
+
+ // This is the name of the table that is in the data file.
+ private static final String TABLENAME = "TestUpgrade";
+
+ // The table has two columns
+ private static final byte [][] TABLENAME_COLUMNS =
+ {Bytes.toBytes("column_a:"), Bytes.toBytes("column_b:")};
- /** {@inheritDoc} */
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- }
+ // Expected count of rows in migrated table.
+ private static final int EXPECTED_COUNT = 17576;
- /** {@inheritDoc} */
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-
/**
+ * @throws IOException
*
*/
- public void testUpgrade() {
+ public void testUpgrade() throws IOException {
MiniDFSCluster dfsCluster = null;
try {
dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
@@ -81,63 +74,46 @@
Path root = dfs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR)));
dfs.mkdirs(root);
- /*
- * First load files from an old style HBase file structure
- */
-
- // Current directory is .../project/build/test/data
-
FileSystem localfs = FileSystem.getLocal(conf);
-
- // Get path for zip file
-
- FSDataInputStream hs = localfs.open(new Path(Path.CUR_DIR,
-
- // this path is for running test with ant
-
- "../../../src/testdata/HADOOP-2478-testdata.zip")
-
- // and this path is for when you want to run inside eclipse
-
- /*"src/testdata/HADOOP-2478-testdata.zip")*/
- );
-
+ // Get path for zip file. If running this test in eclipse, define
+ // the system property src.testdata for your test run.
+ String srcTestdata = System.getProperty("src.testdata");
+ if (srcTestdata == null) {
+ throw new NullPointerException("Define src.test system property");
+ }
+ Path data = new Path(srcTestdata, "HADOOP-2478-testdata.zip");
+ if (!localfs.exists(data)) {
+ throw new FileNotFoundException(data.toString());
+ }
+ FSDataInputStream hs = localfs.open(data);
ZipInputStream zip = new ZipInputStream(hs);
-
unzip(zip, dfs, root);
-
zip.close();
hs.close();
-
listPaths(dfs, root, root.toString().length() + 1);
Migrate u = new Migrate(conf);
u.run(new String[] {"check"});
-
listPaths(dfs, root, root.toString().length() + 1);
u = new Migrate(conf);
u.run(new String[] {"upgrade"});
-
listPaths(dfs, root, root.toString().length() + 1);
// Remove version file and try again
-
dfs.delete(new Path(root, HConstants.VERSION_FILE_NAME));
u = new Migrate(conf);
u.run(new String[] {"upgrade"});
-
listPaths(dfs, root, root.toString().length() + 1);
// Try again. No upgrade should be necessary
-
u = new Migrate(conf);
u.run(new String[] {"check"});
u = new Migrate(conf);
u.run(new String[] {"upgrade"});
-
- } catch (Exception e) {
- e.printStackTrace();
+
+ // Now verify that can read contents.
+ verify();
} finally {
if (dfsCluster != null) {
shutdownDfs(dfsCluster);
@@ -145,14 +121,68 @@
}
}
+ /*
+ * Verify can read the migrated table.
+ * @throws IOException
+ */
+ private void verify() throws IOException {
+ // Delete any cached connections. Need to do this because connection was
+ // created earlier when no master was around. The fact that there was no
+ // master gets cached. Need to delete so we go get master afresh.
+ HConnectionManager.deleteConnection(this.conf);
+ LOG.info("Start a cluster against migrated FS");
+ // Up number of retries. Needed while cluster starts up. Its been set to 1
+ // above.
+ this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER_KEY, 3);
+ MiniHBaseCluster cluster = new MiniHBaseCluster(this.conf, 1);
+ try {
+ HBaseAdmin hb = new HBaseAdmin(this.conf);
+ assertTrue(hb.isMasterRunning());
+ HTableDescriptor [] tables = hb.listTables();
+ boolean foundTable = false;
+ for (int i = 0; i < tables.length; i++) {
+ if (Bytes.equals(Bytes.toBytes(TABLENAME), tables[i].getName())) {
+ foundTable = true;
+ break;
+ }
+ }
+ assertTrue(foundTable);
+ LOG.info(TABLENAME + " exists. Creating an HTable to go against " +
+ TABLENAME + " and master " + this.conf.get(HConstants.MASTER_ADDRESS));
+ HTable t = new HTable(this.conf, TABLENAME);
+ int count = 0;
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ LOG.info("OPENING SCANNER");
+ Scanner s = t.getScanner(TABLENAME_COLUMNS);
+ try {
+ for (RowResult r: s) {
+ if (r == null || r.size() == 0) {
+ break;
+ }
+ count++;
+ if (count % 1000 == 0 && count > 0) {
+ LOG.info("Iterated over " + count + " rows.");
+ }
+ }
+ assertEquals(EXPECTED_COUNT, count);
+ } finally {
+ s.close();
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
private void unzip(ZipInputStream zip, FileSystem dfs, Path root)
throws IOException {
-
ZipEntry e = null;
while ((e = zip.getNextEntry()) != null) {
if (e.isDirectory()) {
dfs.mkdirs(new Path(root, e.getName()));
-
} else {
FSDataOutputStream out = dfs.create(new Path(root, e.getName()));
byte[] buffer = new byte[4096];
@@ -185,4 +215,4 @@
}
}
}
-}
+}
\ No newline at end of file