You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/12/12 00:26:50 UTC
svn commit: r1420482 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/HServerLoad.java
main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
Author: tedyu
Date: Tue Dec 11 23:26:49 2012
New Revision: 1420482
URL: http://svn.apache.org/viewvc?rev=1420482&view=rev
Log:
HBASE-5258 Move coprocessors set out of RegionLoad (Sergey)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1420482&r1=1420481&r2=1420482&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Tue Dec 11 23:26:49 2012
@@ -60,30 +60,14 @@ implements WritableComparable<HServerLoa
private int maxHeapMB = 0;
// Regionserver-level coprocessors, e.g., WALObserver implementations.
- // Region-level coprocessors, on the other hand, are stored inside RegionLoad
- // objects.
- private Set<String> coprocessors =
- new TreeSet<String>();
+ private Set<String> coprocessors = new TreeSet<String>();
/**
* HBASE-4070: Improve region server metrics to report loaded coprocessors.
- *
- * @return Returns the set of all coprocessors on this
- * regionserver, where this set is the union of the
- * regionserver-level coprocessors on one hand, and all of the region-level
- * coprocessors, on the other.
- *
- * We must iterate through all regions loaded on this regionserver to
- * obtain all of the region-level coprocessors.
+ * @return the set of all the server-wide coprocessors on this regionserver
*/
- public String[] getCoprocessors() {
- TreeSet<String> returnValue = new TreeSet<String>(coprocessors);
- for (Map.Entry<byte[], RegionLoad> rls: getRegionsLoad().entrySet()) {
- for (String coprocessor: rls.getValue().getCoprocessors()) {
- returnValue.add(coprocessor);
- }
- }
- return returnValue.toArray(new String[0]);
+ public String[] getRsCoprocessors() {
+ return coprocessors.toArray(new String[0]);
}
/** per-region load metrics */
@@ -145,10 +129,6 @@ implements WritableComparable<HServerLoa
*/
private int totalStaticBloomSizeKB;
- // Region-level coprocessors.
- Set<String> coprocessors =
- new TreeSet<String>();
-
/**
* Constructor, for Writable
*/
@@ -168,7 +148,6 @@ implements WritableComparable<HServerLoa
* @param writeRequestsCount
* @param totalCompactingKVs
* @param currentCompactedKVs
- * @param coprocessors
*/
public RegionLoad(final byte[] name, final int stores,
final int storefiles, final int storeUncompressedSizeMB,
@@ -177,8 +156,7 @@ implements WritableComparable<HServerLoa
final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
final int totalStaticBloomSizeKB,
final long readRequestsCount, final long writeRequestsCount,
- final long totalCompactingKVs, final long currentCompactedKVs,
- final Set<String> coprocessors) {
+ final long totalCompactingKVs, final long currentCompactedKVs) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
@@ -193,12 +171,6 @@ implements WritableComparable<HServerLoa
this.writeRequestsCount = writeRequestsCount;
this.totalCompactingKVs = totalCompactingKVs;
this.currentCompactedKVs = currentCompactedKVs;
- this.coprocessors = coprocessors;
- }
-
- // Getters
- private String[] getCoprocessors() {
- return coprocessors.toArray(new String[0]);
}
/**
@@ -400,9 +372,9 @@ implements WritableComparable<HServerLoa
this.totalCompactingKVs = in.readLong();
this.currentCompactedKVs = in.readLong();
int coprocessorsSize = in.readInt();
- coprocessors = new TreeSet<String>();
+ // Backward compatibility - there may be coprocessors in the region load, ignore them.
for (int i = 0; i < coprocessorsSize; i++) {
- coprocessors.add(in.readUTF());
+ in.readUTF();
}
}
@@ -431,9 +403,9 @@ implements WritableComparable<HServerLoa
this.totalCompactingKVs = WritableUtils.readVLong(in);
this.currentCompactedKVs = WritableUtils.readVLong(in);
int coprocessorsSize = WritableUtils.readVInt(in);
- coprocessors = new TreeSet<String>();
+ // Backward compatibility - there may be coprocessors in the region load, ignore them.
for (int i = 0; i < coprocessorsSize; i++) {
- coprocessors.add(in.readUTF());
+ in.readUTF();
}
}
@@ -454,10 +426,9 @@ implements WritableComparable<HServerLoa
WritableUtils.writeVInt(out, totalStaticBloomSizeKB);
WritableUtils.writeVLong(out, totalCompactingKVs);
WritableUtils.writeVLong(out, currentCompactedKVs);
- WritableUtils.writeVInt(out, coprocessors.size());
- for (String coprocessor: coprocessors) {
- out.writeUTF(coprocessor);
- }
+ // Backward compatibility - write out 0 as coprocessor count,
+ // we don't report region-level coprocessors anymore.
+ WritableUtils.writeVInt(out, 0);
}
/**
@@ -503,11 +474,6 @@ implements WritableComparable<HServerLoa
}
sb = Strings.appendKeyValue(sb, "compactionProgressPct",
compactionProgressPct);
- String coprocessors = Arrays.toString(getCoprocessors());
- if (coprocessors != null) {
- sb = Strings.appendKeyValue(sb, "coprocessors",
- Arrays.toString(getCoprocessors()));
- }
return sb.toString();
}
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1420482&r1=1420481&r2=1420482&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Dec 11 23:26:49 2012
@@ -44,6 +44,7 @@ import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -1096,8 +1097,7 @@ public class HRegionServer implements HR
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, rootIndexSizeKB,
totalStaticIndexSizeKB, totalStaticBloomSizeKB,
(int) r.readRequestsCount.get(), (int) r.writeRequestsCount.get(),
- totalCompactingKVs, currentCompactedKVs,
- r.getCoprocessorHost().getCoprocessors());
+ totalCompactingKVs, currentCompactedKVs);
}
/**
@@ -3776,8 +3776,13 @@ public class HRegionServer implements HR
// used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
public String[] getCoprocessors() {
- HServerLoad hsl = buildServerLoad();
- return hsl == null? null: hsl.getCoprocessors();
+ TreeSet<String> coprocessors = new TreeSet<String>(
+ this.hlog.getCoprocessorHost().getCoprocessors());
+ Collection<HRegion> regions = getOnlineRegionsLocalContext();
+ for (HRegion region: regions) {
+ coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
+ }
+ return coprocessors.toArray(new String[0]);
}
/**
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java?rev=1420482&r1=1420481&r2=1420482&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java Tue Dec 11 23:26:49 2012
@@ -68,14 +68,6 @@ public class TestClassLoading {
private static final String[] regionServerSystemCoprocessors =
new String[]{
- regionCoprocessor1.getSimpleName(),
- regionServerCoprocessor.getSimpleName()
- };
-
- private static final String[] regionServerSystemAndUserCoprocessors =
- new String[] {
- regionCoprocessor1.getSimpleName(),
- regionCoprocessor2.getSimpleName(),
regionServerCoprocessor.getSimpleName()
};
@@ -424,6 +416,8 @@ public class TestClassLoading {
File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
byte buffer[] = new byte[BUFFER_SIZE];
+ // TODO: code here and elsewhere in this file is duplicated w/TestClassFinder.
+ // Some refactoring may be in order...
// Open archive file
FileOutputStream stream = new FileOutputStream(outerJarFile);
JarOutputStream out = new JarOutputStream(stream, new Manifest());
@@ -433,7 +427,7 @@ public class TestClassLoading {
JarEntry jarAdd = new JarEntry("/lib/" + jarFile.getName());
jarAdd.setTime(jarFile.lastModified());
out.putNextEntry(jarAdd);
-
+
// Write file to archive
FileInputStream in = new FileInputStream(jarFile);
while (true) {
@@ -505,82 +499,12 @@ public class TestClassLoading {
@Test
public void testRegionServerCoprocessorsReported() throws Exception {
- // HBASE 4070: Improve region server metrics to report loaded coprocessors
- // to master: verify that each regionserver is reporting the correct set of
- // loaded coprocessors.
-
- // We rely on the fact that getCoprocessors() will return a sorted
- // display of the coprocessors' names, so for example, regionCoprocessor1's
- // name "ColumnAggregationEndpoint" will appear before regionCoprocessor2's
- // name "GenericEndpoint" because "C" is before "G" lexicographically.
+ // This was a test for HBASE-4070.
+ // We are removing coprocessors from region load in HBASE-5258.
+ // Therefore, this test now only checks system coprocessors.
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
-
- // disable all user tables, if any are loaded.
- for (HTableDescriptor htd: admin.listTables()) {
- if (!htd.isMetaTable()) {
- String tableName = htd.getNameAsString();
- if (admin.isTableEnabled(tableName)) {
- try {
- admin.disableTable(htd.getNameAsString());
- } catch (TableNotEnabledException e) {
- // ignoring this exception for now : not sure why it's happening.
- }
- }
- }
- }
-
- // should only be system coprocessors loaded at this point.
- assertAllRegionServers(regionServerSystemCoprocessors,null);
-
- // The next two tests enable and disable user tables to see if coprocessor
- // load reporting changes as coprocessors are loaded and unloaded.
- //
-
- // Create a table.
- // should cause regionCoprocessor2 to be loaded, since we've specified it
- // for loading on any user table with USER_REGION_COPROCESSOR_CONF_KEY
- // in setUpBeforeClass().
- String userTable1 = "userTable1";
- HTableDescriptor userTD1 = new HTableDescriptor(userTable1);
- admin.createTable(userTD1);
- waitForTable(userTD1.getName());
-
- // table should be enabled now.
- assertTrue(admin.isTableEnabled(userTable1));
- assertAllRegionServers(regionServerSystemAndUserCoprocessors, userTable1);
-
- // unload and make sure we're back to only system coprocessors again.
- admin.disableTable(userTable1);
assertAllRegionServers(regionServerSystemCoprocessors,null);
-
- // create another table, with its own specified coprocessor.
- String userTable2 = "userTable2";
- HTableDescriptor htd2 = new HTableDescriptor(userTable2);
-
- String userTableCP = "userTableCP";
- File jarFile1 = buildCoprocessorJar(userTableCP);
- htd2.addFamily(new HColumnDescriptor("myfamily"));
- htd2.setValue("COPROCESSOR$1", jarFile1.toString() + "|" + userTableCP +
- "|" + Coprocessor.PRIORITY_USER);
- admin.createTable(htd2);
- waitForTable(htd2.getName());
- // table should be enabled now.
- assertTrue(admin.isTableEnabled(userTable2));
-
- ArrayList<String> existingCPsPlusNew =
- new ArrayList<String>(Arrays.asList(regionServerSystemAndUserCoprocessors));
- existingCPsPlusNew.add(userTableCP);
- String[] existingCPsPlusNewArray = new String[existingCPsPlusNew.size()];
- assertAllRegionServers(existingCPsPlusNew.toArray(existingCPsPlusNewArray),
- userTable2);
-
- admin.disableTable(userTable2);
- assertTrue(admin.isTableDisabled(userTable2));
-
- // we should be back to only system coprocessors again.
- assertAllRegionServers(regionServerSystemCoprocessors, null);
-
}
/**
@@ -627,7 +551,7 @@ public class TestClassLoading {
}
boolean any_failed = false;
for(Map.Entry<ServerName,HServerLoad> server: servers.entrySet()) {
- actualCoprocessors = server.getValue().getCoprocessors();
+ actualCoprocessors = server.getValue().getRsCoprocessors();
if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
LOG.debug("failed comparison: actual: " +
Arrays.toString(actualCoprocessors) +