You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2013/08/20 07:26:52 UTC
svn commit: r1515706 - in /hbase/branches/0.92/src:
main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
Author: apurtell
Date: Tue Aug 20 05:26:52 2013
New Revision: 1515706
URL: http://svn.apache.org/r1515706
Log:
HBASE-9212. [0.92] TestMasterObserver fails occasionally
Modified:
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java?rev=1515706&r1=1515705&r2=1515706&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java Tue Aug 20 05:26:52 2013
@@ -155,6 +155,10 @@ public class FSTableDescriptors implemen
if (tdm != null) {
if (modtime <= tdm.getModtime()) {
cachehits++;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Hit: modtime=" + modtime + " cached=" + tdm.getModtime() + " htd=" +
+ tdm.getTableDescriptor());
+ }
return tdm.getTableDescriptor();
}
}
@@ -175,6 +179,9 @@ public class FSTableDescriptors implemen
"doesn't contain a table descriptor, " +
"do consider deleting it: " + tablename);
} else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Miss: modtime=" + modtime + " htd=" + htd);
+ }
this.cache.put(tablename, new TableDescriptorModtime(modtime, htd));
}
return htd;
@@ -214,8 +221,15 @@ public class FSTableDescriptors implemen
if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) {
throw new NotImplementedException();
}
- if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd);
- long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString());
+ if (fsreadonly) {
+ // Cannot cache here.
+ // We can't know if a modtime from the most recent file found in a
+ // directory listing at some arbitrary point in time still corresponds
+ // to the latest, nor that our htd is the latest.
+ return;
+ }
+ // Cache with the modtime of the descriptor we wrote
+ long modtime = updateHTableDescriptor(this.fs, this.rootdir, htd).getModificationTime();
this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd));
}
@@ -419,6 +433,9 @@ public class FSTableDescriptors implemen
} finally {
fsDataInputStream.close();
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Read table descriptor from " + status.getPath() + ": " + hTableDescriptor);
+ }
return hTableDescriptor;
}
@@ -430,7 +447,7 @@ public class FSTableDescriptors implemen
* @return New tableinfo or null if we failed update.
* @throws IOException Thrown if failed update.
*/
- static Path updateHTableDescriptor(FileSystem fs, Path rootdir,
+ static FileStatus updateHTableDescriptor(FileSystem fs, Path rootdir,
HTableDescriptor hTableDescriptor)
throws IOException {
Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName());
@@ -438,7 +455,11 @@ public class FSTableDescriptors implemen
getTableInfoPath(fs, tableDir));
if (p == null) throw new IOException("Failed update");
LOG.info("Updated tableinfo=" + p);
- return p;
+ FileStatus[] status = FSUtils.listStatus(fs, p, null);
+ if (status == null || status.length != 1) {
+ throw new IOException("listStatus failed to return expected result");
+ }
+ return status[0];
}
/**
Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java?rev=1515706&r1=1515705&r2=1515706&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java Tue Aug 20 05:26:52 2013
@@ -31,8 +31,6 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
-import junit.framework.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -470,7 +468,6 @@ public class TestMasterObserver {
}
private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static byte[] TEST_TABLE = Bytes.toBytes("observed_table");
private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
@@ -488,11 +485,14 @@ public class TestMasterObserver {
UTIL.shutdownMiniCluster();
}
- @Test
+ @Test(timeout=60000)
public void testStarted() throws Exception {
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
HMaster master = cluster.getMaster();
+ while (!master.isInitialized()) {
+ Thread.sleep(100);
+ }
assertTrue("Master should be active", master.isActiveMaster());
MasterCoprocessorHost host = master.getCoprocessorHost();
assertNotNull("CoprocessorHost should not be null", host);
@@ -506,8 +506,9 @@ public class TestMasterObserver {
cp.wasStartMasterCalled());
}
- @Test
+ @Test(timeout=120000)
public void testTableOperations() throws Exception {
+ byte[] tableName = Bytes.toBytes("testTableOperations");
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
HMaster master = cluster.getMaster();
@@ -519,107 +520,108 @@ public class TestMasterObserver {
assertFalse("No table created yet", cp.wasCreateTableCalled());
// create a table
- HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
+ HTableDescriptor htd = new HTableDescriptor(tableName);
htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
HBaseAdmin admin = UTIL.getHBaseAdmin();
admin.createTable(htd);
+ UTIL.waitTableAvailable(htd.getName(), 5000);
+
// preCreateTable can't bypass default action.
assertTrue("Test table should be created", cp.wasCreateTableCalled());
- admin.disableTable(TEST_TABLE);
- assertTrue(admin.isTableDisabled(TEST_TABLE));
+ admin.disableTable(tableName);
+ assertTrue(admin.isTableDisabled(tableName));
// preDisableTable can't bypass default action.
assertTrue("Coprocessor should have been called on table disable",
cp.wasDisableTableCalled());
// enable
assertFalse(cp.wasEnableTableCalled());
- admin.enableTable(TEST_TABLE);
- assertTrue(admin.isTableEnabled(TEST_TABLE));
+ admin.enableTable(tableName);
+ assertTrue(admin.isTableEnabled(tableName));
// preEnableTable can't bypass default action.
assertTrue("Coprocessor should have been called on table enable",
cp.wasEnableTableCalled());
- admin.disableTable(TEST_TABLE);
- assertTrue(admin.isTableDisabled(TEST_TABLE));
+ admin.disableTable(tableName);
+ assertTrue(admin.isTableDisabled(tableName));
// modify table
htd.setMaxFileSize(512 * 1024 * 1024);
- admin.modifyTable(TEST_TABLE, htd);
+ admin.modifyTable(tableName, htd);
// preModifyTable can't bypass default action.
assertTrue("Test table should have been modified",
cp.wasModifyTableCalled());
// add a column family
- admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
+ admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
assertTrue("New column family shouldn't have been added to test table",
cp.preAddColumnCalledOnly());
// modify a column family
HColumnDescriptor hcd1 = new HColumnDescriptor(TEST_FAMILY2);
hcd1.setMaxVersions(25);
- admin.modifyColumn(TEST_TABLE, hcd1);
+ admin.modifyColumn(tableName, hcd1);
assertTrue("Second column family should be modified",
cp.preModifyColumnCalledOnly());
// delete table
- admin.deleteTable(TEST_TABLE);
+ admin.deleteTable(tableName);
assertFalse("Test table should have been deleted",
- admin.tableExists(TEST_TABLE));
+ admin.tableExists(tableName));
// preDeleteTable can't bypass default action.
assertTrue("Coprocessor should have been called on table delete",
cp.wasDeleteTableCalled());
-
// turn off bypass, run the tests again
cp.enableBypass(false);
cp.resetStates();
admin.createTable(htd);
+ UTIL.waitTableAvailable(htd.getName(), 5000);
assertTrue("Test table should be created", cp.wasCreateTableCalled());
// disable
assertFalse(cp.wasDisableTableCalled());
-
- admin.disableTable(TEST_TABLE);
- assertTrue(admin.isTableDisabled(TEST_TABLE));
+ admin.disableTable(tableName);
+ assertTrue(admin.isTableDisabled(tableName));
assertTrue("Coprocessor should have been called on table disable",
cp.wasDisableTableCalled());
// modify table
htd.setMaxFileSize(512 * 1024 * 1024);
- admin.modifyTable(TEST_TABLE, htd);
+ admin.modifyTable(tableName, htd);
assertTrue("Test table should have been modified",
cp.wasModifyTableCalled());
// add a column family
- admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
+ admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
assertTrue("New column family should have been added to test table",
cp.wasAddColumnCalled());
// modify a column family
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
hcd.setMaxVersions(25);
- admin.modifyColumn(TEST_TABLE, hcd);
+ admin.modifyColumn(tableName, hcd);
assertTrue("Second column family should be modified",
cp.wasModifyColumnCalled());
// enable
assertFalse(cp.wasEnableTableCalled());
- admin.enableTable(TEST_TABLE);
- assertTrue(admin.isTableEnabled(TEST_TABLE));
+ admin.enableTable(tableName);
+ assertTrue(admin.isTableEnabled(tableName));
assertTrue("Coprocessor should have been called on table enable",
cp.wasEnableTableCalled());
// disable again
- admin.disableTable(TEST_TABLE);
- assertTrue(admin.isTableDisabled(TEST_TABLE));
+ admin.disableTable(tableName);
+ assertTrue(admin.isTableDisabled(tableName));
// delete column
assertFalse("No column family deleted yet", cp.wasDeleteColumnCalled());
- admin.deleteColumn(TEST_TABLE, TEST_FAMILY2);
- HTableDescriptor tableDesc = admin.getTableDescriptor(TEST_TABLE);
+ admin.deleteColumn(tableName, TEST_FAMILY2);
+ HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
assertNull("'"+Bytes.toString(TEST_FAMILY2)+"' should have been removed",
tableDesc.getFamily(TEST_FAMILY2));
assertTrue("Coprocessor should have been called on column delete",
@@ -627,15 +629,16 @@ public class TestMasterObserver {
// delete table
assertFalse("No table deleted yet", cp.wasDeleteTableCalled());
- admin.deleteTable(TEST_TABLE);
+ admin.deleteTable(tableName);
assertFalse("Test table should have been deleted",
- admin.tableExists(TEST_TABLE));
+ admin.tableExists(tableName));
assertTrue("Coprocessor should have been called on table delete",
cp.wasDeleteTableCalled());
}
- @Test
+ @Test(timeout=120000)
public void testRegionTransitionOperations() throws Exception {
+ byte[] tableName = Bytes.toBytes("testRegionTransitionOperations");
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
HMaster master = cluster.getMaster();
@@ -645,7 +648,7 @@ public class TestMasterObserver {
cp.enableBypass(false);
cp.resetStates();
- HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
+ HTable table = UTIL.createTable(tableName, TEST_FAMILY);
int countOfRegions = UTIL.createMultiRegions(table, TEST_FAMILY);
UTIL.waitUntilAllRegionsAssigned(countOfRegions);
Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java?rev=1515706&r1=1515705&r2=1515706&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java Tue Aug 20 05:26:52 2013
@@ -78,14 +78,14 @@ public class TestFSTableDescriptors {
Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
HTableDescriptor htd = new HTableDescriptor("testSequenceidAdvancesOnTableInfo");
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
- Path p0 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd);
+ Path p0 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd).getPath();
int i0 = FSTableDescriptors.getTableInfoSequenceid(p0);
- Path p1 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd);
+ Path p1 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd).getPath();
// Assert we cleaned up the old file.
assertTrue(!fs.exists(p0));
int i1 = FSTableDescriptors.getTableInfoSequenceid(p1);
assertTrue(i1 == i0 + 1);
- Path p2 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd);
+ Path p2 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd).getPath();
// Assert we cleaned up the old file.
assertTrue(!fs.exists(p1));
int i2 = FSTableDescriptors.getTableInfoSequenceid(p2);