You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/02/29 22:05:59 UTC
svn commit: r1295272 - in /incubator/accumulo/trunk: ./ src/core/
src/packages/deb/accumulo-native/ src/packages/deb/accumulo/ src/server/
src/server/src/main/java/org/apache/accumulo/server/master/
src/server/src/main/java/org/apache/accumulo/server/m...
Author: kturner
Date: Wed Feb 29 21:05:58 2012
New Revision: 1295272
URL: http://svn.apache.org/viewvc?rev=1295272&view=rev
Log:
ACCUMULO-436 ACCUMULO-19 ACCUMULO-431 merge from 1.4
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/packages/deb/accumulo-native/control
incubator/accumulo/trunk/src/packages/deb/accumulo/control
incubator/accumulo/trunk/src/server/ (props changed)
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml
incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 29 21:05:58 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1294846
+/incubator/accumulo/branches/1.4:1201902-1295271
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 29 21:05:58 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
/incubator/accumulo/branches/1.3.5rc/src/core:1209938
-/incubator/accumulo/branches/1.4/src/core:1201902-1294805
+/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
+/incubator/accumulo/branches/1.4/src/core:1201902-1295271
Modified: incubator/accumulo/trunk/src/packages/deb/accumulo-native/control
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/packages/deb/accumulo-native/control?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/packages/deb/accumulo-native/control (original)
+++ incubator/accumulo/trunk/src/packages/deb/accumulo-native/control Wed Feb 29 21:05:58 2012
@@ -21,6 +21,6 @@ Package: accumulo-native
Version: [[version]]
Section: misc
Architecture: all
-Depends: default-jdk, accumulo (=[[version]]), g++, g++-multilib
+Depends: java6-sdk, accumulo (=[[version]]), g++, g++-multilib
Description: Native libraries for Accumulo- locks and InMemoryMap
This optional package contains native libraries that improve the performance of Accumulo tservers.
Modified: incubator/accumulo/trunk/src/packages/deb/accumulo/control
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/packages/deb/accumulo/control?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/packages/deb/accumulo/control (original)
+++ incubator/accumulo/trunk/src/packages/deb/accumulo/control Wed Feb 29 21:05:58 2012
@@ -21,7 +21,7 @@ Package: accumulo
Version: [[version]]
Section: misc
Architecture: all
-Depends: default-jre-headless, hadoop, hadoop-zookeeper, coreutils
+Depends: java6-runtime, hadoop, hadoop-zookeeper, coreutils, ssh-server
Recommends: accumulo-native
Description: Accumulo is a large distributed structured store based on Google's BigTable design.
Distribution: development
Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 29 21:05:58 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
/incubator/accumulo/branches/1.3.5rc/src/server:1209938
-/incubator/accumulo/branches/1.4/src/server:1201902-1294805
+/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
+/incubator/accumulo/branches/1.4/src/server:1201902-1295271
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java Wed Feb 29 21:05:58 2012
@@ -41,11 +41,11 @@ import java.util.concurrent.atomic.Atomi
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -1506,26 +1506,30 @@ public class Master implements LiveTServ
private void updateMergeState(Map<Text,MergeStats> mergeStatsCache) {
for (MergeStats stats : mergeStatsCache.values()) {
try {
- MergeState update = stats.nextMergeState();
- if (update == MergeState.MERGING) {
- if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
- try {
- if (stats.getMergeInfo().isDelete()) {
- deleteTablets(stats.getMergeInfo());
- } else {
- mergeMetadataRecords(stats.getMergeInfo());
- }
- setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
- } catch (Exception ex) {
- log.error("Unable merge metadata table records", ex);
- }
- }
- }
+ MergeState update = stats.nextMergeState(getConnector(), Master.this);
+
+ // when next state is MERGING, its important to persist this before
+ // starting the merge... the verification check that is done before
+ // moving into the merging state could fail if merge starts but does
+ // not finish
if (update == MergeState.COMPLETE)
update = MergeState.NONE;
if (update != stats.getMergeInfo().getState()) {
setMergeState(stats.getMergeInfo(), update);
}
+
+ if (update == MergeState.MERGING) {
+ try {
+ if (stats.getMergeInfo().isDelete()) {
+ deleteTablets(stats.getMergeInfo());
+ } else {
+ mergeMetadataRecords(stats.getMergeInfo());
+ }
+ setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
+ } catch (Exception ex) {
+ log.error("Unable merge metadata table records", ex);
+ }
+ }
} catch (Exception ex) {
log.error("Unable to update merge state for merge " + stats.getMergeInfo().getRange(), ex);
}
@@ -1579,18 +1583,21 @@ public class Master implements LiveTServ
}
}
MetadataTable.addDeleteEntries(range, datafiles, SecurityConstants.getSystemCredentials());
- log.debug("Removing metadata table entries in range " + deleteRange);
- BatchDeleter bd = conn.createBatchDeleter(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 4, 100000l, 1000l, 4);
- bd.setRanges(Collections.singleton(deleteRange));
- bd.delete();
- bd.close();
+ BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000000l, 100l, 1);
+ try {
+ deleteTablets(deleteRange, bw, conn);
+ } finally {
+ bw.close();
+ }
+
if (followingTablet != null) {
log.debug("Updating prevRow of " + followingTablet + " to " + range.getPrevEndRow());
- BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000l, 100l, 1);
+ bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000l, 100l, 1);
try {
Mutation m = new Mutation(followingTablet.getMetadataEntry());
ColumnFQ.put(m, Constants.METADATA_PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(range.getPrevEndRow()));
+ ColumnFQ.putDelete(m, Constants.METADATA_CHOPPED_COLUMN);
bw.addMutation(m);
bw.flush();
} finally {
@@ -1618,7 +1625,7 @@ public class Master implements LiveTServ
if (start == null) {
start = new Text();
}
- Range scanRange = new Range(KeyExtent.getMetadataEntry(range.getTableId(), start), false, stopRow, true);
+ Range scanRange = new Range(KeyExtent.getMetadataEntry(range.getTableId(), start), false, stopRow, false);
BatchWriter bw = null;
try {
long fileCount = 0;
@@ -1629,14 +1636,13 @@ public class Master implements LiveTServ
scanner.setRange(scanRange);
ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
ColumnFQ.fetch(scanner, Constants.METADATA_TIME_COLUMN);
+ ColumnFQ.fetch(scanner, Constants.METADATA_DIRECTORY_COLUMN);
scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
Mutation m = new Mutation(stopRow);
String maxLogicalTime = null;
for (Entry<Key,Value> entry : scanner) {
Key key = entry.getKey();
Value value = entry.getValue();
- if (key.getRow().equals(stopRow))
- break;
if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
m.put(key.getColumnFamily(), key.getColumnQualifier(), value);
fileCount++;
@@ -1645,6 +1651,8 @@ public class Master implements LiveTServ
firstPrevRowValue = new Value(value);
} else if (Constants.METADATA_TIME_COLUMN.hasColumns(key)) {
maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, value.toString());
+ } else if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+ bw.addMutation(MetadataTable.createDeleteMutation(range.getTableId().toString(), entry.getValue().toString()));
}
}
@@ -1663,8 +1671,10 @@ public class Master implements LiveTServ
if (!m.getUpdates().isEmpty()) {
bw.addMutation(m);
- bw.flush();
}
+
+ bw.flush();
+
log.debug("Moved " + fileCount + " files to " + stop);
if (firstPrevRowValue == null) {
@@ -1677,38 +1687,13 @@ public class Master implements LiveTServ
log.debug("Setting the prevRow for last tablet: " + stop);
bw.addMutation(updatePrevRow);
bw.flush();
-
- // Delete everything in the other tablets
- scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
- log.debug("Scanning range " + scanRange);
- scanner.setRange(scanRange);
- for (Entry<Key,Value> entry : scanner) {
- Key key = entry.getKey();
- if (key.getRow().equals(stopRow))
- break;
- if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
- bw.addMutation(MetadataTable.createDeleteMutation(range.getTableId().toString(), entry.getValue().toString()));
- }
-
- // TODO could group by row
- m = new Mutation(key.getRow());
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
- log.debug("deleting entry " + key);
- bw.addMutation(m);
- }
- bw.flush();
+
+ deleteTablets(scanRange, bw, conn);
// Clean-up the last chopped marker
- scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
- scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
- scanner.setRange(new Range(stopRow, stopRow));
- for (Entry<Key,Value> entry : scanner) {
- Key key = entry.getKey();
- m = new Mutation(key.getRow());
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
- log.debug("deleting entry " + key);
- bw.addMutation(m);
- }
+ m = new Mutation(stopRow);
+ ColumnFQ.putDelete(m, Constants.METADATA_CHOPPED_COLUMN);
+ bw.addMutation(m);
bw.flush();
} catch (Exception ex) {
@@ -1721,6 +1706,36 @@ public class Master implements LiveTServ
}
}
}
+
+ private void deleteTablets(Range scanRange, BatchWriter bw, Connector conn) throws TableNotFoundException, MutationsRejectedException {
+ Scanner scanner;
+ Mutation m;
+ // Delete everything in the other tablets
+ // group all deletes into tablet into one mutation, this makes tablets
+ // either dissapear entirely or not all.. this is important for the case
+ // where the process terminates in the loop below...
+ scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ log.debug("Deleting range " + scanRange);
+ scanner.setRange(scanRange);
+ RowIterator rowIter = new RowIterator(scanner);
+ while (rowIter.hasNext()) {
+ Iterator<Entry<Key,Value>> row = rowIter.next();
+ m = null;
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+
+ if (m == null)
+ m = new Mutation(key.getRow());
+
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+ log.debug("deleting entry " + key);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.flush();
+ }
private KeyExtent getHighTablet(KeyExtent range) throws AccumuloException {
try {
@@ -1734,7 +1749,11 @@ public class Master implements LiveTServ
throw new AccumuloException("No last tablet for a merge " + range);
}
Entry<Key,Value> entry = iterator.next();
- return new KeyExtent(entry.getKey().getRow(), KeyExtent.decodePrevEndRow(entry.getValue()));
+ KeyExtent highTablet = new KeyExtent(entry.getKey().getRow(), KeyExtent.decodePrevEndRow(entry.getValue()));
+ if (highTablet.getTableId() != range.getTableId()) {
+ throw new AccumuloException("No last tablet for merge " + range + " " + highTablet);
+ }
+ return highTablet;
} catch (Exception ex) {
throw new AccumuloException("Unexpected failure finding the last tablet for a merge " + range, ex);
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java Wed Feb 29 21:05:58 2012
@@ -93,7 +93,7 @@ public class MergeStats {
this.unassigned++;
}
- public MergeState nextMergeState() throws Exception {
+ public MergeState nextMergeState(Connector connector, CurrentState master) throws Exception {
MergeState state = info.getState();
if (state == MergeState.NONE)
return state;
@@ -141,7 +141,10 @@ public class MergeStats {
} else {
log.info(chopped + " tablets are chopped, " + unassigned + " are offline " + info.getRange());
if (unassigned == total && chopped == needsToBeChopped) {
- state = MergeState.MERGING;
+ if (verifyMergeConsistency(connector, master))
+ state = MergeState.MERGING;
+ else
+ log.info("Merge consistency check failed " + info.getRange());
} else {
log.info("Waiting for " + unassigned + " unassigned tablets to be " + total + " " + info.getRange());
}
@@ -163,7 +166,7 @@ public class MergeStats {
return state;
}
- public boolean verifyMergeConsistency(Connector connector, CurrentState master) throws TableNotFoundException, IOException {
+ private boolean verifyMergeConsistency(Connector connector, CurrentState master) throws TableNotFoundException, IOException {
MergeStats verify = new MergeStats(info);
Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
MetaDataTableScanner.configureScanner(scanner, master);
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java Wed Feb 29 21:05:58 2012
@@ -161,6 +161,7 @@ abstract public class BasicServlet exten
sb.append("<a href='/master'>Master Server</a><br />\n");
sb.append("<a href='/tservers'>Tablet Servers</a><br />\n");
sb.append("<a href='/loggers'>Logger Servers</a><br />\n");
+ sb.append("<a href='/vis'>Server Activity</a><br />\n");
sb.append("<a href='/gc'>Garbage Collector</a><br />\n");
sb.append("<a href='/tables'>Tables</a><br />\n");
sb.append("<a href='/trace/summary?minutes=10'>Recent Traces</a><br />\n");
@@ -172,7 +173,6 @@ abstract public class BasicServlet exten
if (numProblems > 0)
sb.append("<span class='error'><a href='/problems'>Table Problems <span class='smalltext'>(" + numProblems + ")</a></span></span><br />\n");
sb.append("<hr />\n");
- sb.append("<a href='/vis'>Visualization</a><br />\n");
sb.append("<a href='/xml'>XML</a><hr />\n");
sb.append("<div class='smalltext'>[<a href='").append("/op?action=refresh&value=").append(refresh < 1 ? "5" : "-1");
sb.append("&redir=").append(currentPage(req)).append("'>");
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java Wed Feb 29 21:05:58 2012
@@ -30,13 +30,18 @@ import org.apache.accumulo.server.monito
public class VisServlet extends BasicServlet {
private static final long serialVersionUID = 1L;
boolean useCircles;
- boolean useIngest;
+ StatType motion;
+ StatType color;
int spacing;
String url;
+ public enum StatType {
+ osload, ingest, query
+ }
+
@Override
protected String getTitle(HttpServletRequest req) {
- return "Tablet Server Status Visualization";
+ return "Server Activity";
}
@Override
@@ -46,15 +51,25 @@ public class VisServlet extends BasicSer
url = urlsb.toString();
useCircles = true;
- String shape = req.getParameter("shape");
- if (shape != null && (shape.equals("square") || shape.equals("squares"))) {
+ String s = req.getParameter("shape");
+ if (s != null && (s.equals("square") || s.equals("squares"))) {
useCircles = false;
}
- useIngest = true;
- String motion = req.getParameter("motion");
- if (motion != null && (motion.equals("query"))) {
- useIngest = false;
+ s = req.getParameter("motion");
+ motion = StatType.ingest;
+ if (s != null) {
+ try {
+ motion = StatType.valueOf(s);
+ } catch (Exception e) {}
+ }
+
+ s = req.getParameter("color");
+ color = StatType.osload;
+ if (s != null) {
+ try {
+ color = StatType.valueOf(s);
+ } catch (Exception e) {}
}
spacing = 20;
@@ -64,6 +79,8 @@ public class VisServlet extends BasicSer
spacing = 10;
else if (size.equals("40"))
spacing = 40;
+ else if (size.equals("80"))
+ spacing = 80;
}
ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
@@ -88,12 +105,16 @@ public class VisServlet extends BasicSer
// size select box
sb.append("  <span class='viscontrol'>Size: <select id='size' onchange='setSize(this)'><option").append(spacing == 10 ? " selected='true'" : "")
.append(">10</option><option").append(spacing == 20 ? " selected='true'" : "").append(">20</option><option")
- .append(spacing == 40 ? " selected='true'" : "").append(">40</option></select></span>\n");
+ .append(spacing == 40 ? " selected='true'" : "").append(">40</option><option").append(spacing == 80 ? " selected='true'" : "")
+ .append(">80</option></select></span>\n");
// motion select box
- sb.append("  <span class='viscontrol'>Motion: <select id='motion' onchange='setMotion(this)'><option>Ingest</option><option")
- .append(!useIngest ? " selected='true'" : "").append(">Query</option></select></span>\n");
+ sb.append("  <span class='viscontrol'>Motion: <select id='motion' onchange='setMotion(this)'>");
+ addOptions(sb, motion);
+ sb.append("</select></span>\n");
// color select box
- sb.append("  <span class='viscontrol'>Color: <select><option>OS Load</option></select></span>\n");
+ sb.append("  <span class='viscontrol'>Color: <select id='color' onchange='setColor(this)'>");
+ addOptions(sb, color);
+ sb.append("</select></span>\n");
sb.append("  <span class='viscontrol'>(hover for info, click for details)</span>");
sb.append("</div>\n\n");
// floating info box
@@ -110,6 +131,12 @@ public class VisServlet extends BasicSer
sb.append("</script>\n");
}
+ private void addOptions(StringBuilder sb, StatType selectedStatType) {
+ for (StatType st : StatType.values()) {
+ sb.append("<option").append(st.equals(selectedStatType) ? " selected='true'>" : ">").append(st).append("</option>");
+ }
+ }
+
private void doScript(StringBuilder sb, ArrayList<TabletServerStatus> tservers) {
InputStream data = VisServlet.class.getClassLoader().getResourceAsStream("web/vis.xml");
if (data != null) {
Modified: incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml (original)
+++ incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml Wed Feb 29 21:05:58 2012
@@ -16,12 +16,18 @@
-->
<script type='text/javascript'>
-var maxLoad = 3*numCores;
-var maxThroughput = 7; // exponent of 10
+// observable stats that can be connected to motion or color
+var statName = ['osload','ingest','query'];
+var speedStatType = 1; // index into statName
+var colorStatType = 0; // index into statName
+var maxStatValue = [numCores, 1000, 10000]; // initial values that are system-dependent will increase based on observed values
+var significance = [100,1,1]; // values will be converted by floor(this*value)/this
+var maxObservedSpeed = maxStatValue[speedStatType];
+var maxObservedColor = maxStatValue[colorStatType];
// size and spacing variables
var numDots = 0; // number of dots to draw
-var dotSpacing = 10; // spacing between centers of dots
+var dotSpacing = 10; // spacing between centers of dots (radius)
var dotPadding = 0.5; // dot padding
var minDotRadius = 3; // min dot radius
var maxDotRadius = dotSpacing - dotPadding;
@@ -29,12 +35,11 @@ var maxDotRadius = dotSpacing - dotPaddi
// arrays of information about each dot
var dotSize = new Array(numDots); // current sizes
var dotSizeGrowing = new Array(numDots); // true when dot size is growing, false when shrinking
-var resizeFrameModulus = new Array(numDots); // never resize when <= 0, otherwise resize when frame % modulus == 0
var ids = new Array(numDots); // server ids
var extras = new Array(numDots); // info from which color and motion is derived
var mousedDot = 0; // the dot currently under the mouse
-var maxObservedColor = maxLoad;
+var speed = new Array(numDots); // between 0 (motionless) and maxObservedSpeed (fastest)
var colors = new Array(numDots); // dot colors between 0 and maxObservedColor, -1 for dead node
var colorPalette = ['#0000CC', '#0014B8', '#0029A3', '#003D8F', '#00527A', '#006666', '#007A52', '#008F3D', '#00A329', '#00B814', '#00CC00', '#14D100', '#29D600', '#3DDB00', '#52E000', '#66E600', '#7AEB00', '#8FF000', '#A3F500', '#B8FA00', '#CCFF00', '#CCFF00', '#CCF200', '#CCE600', '#CCD900', '#CCCC00', '#CCBF00', '#CCB200', '#CCA600', '#CC9900', '#CC8C00', '#CC8000', '#CC7300', '#CC6600', '#CC5900', '#CC4C00', '#CC4000', '#CC3300', '#CC2600', '#CC1A00', '#CC0D00', '#CC0000'];
@@ -55,7 +60,6 @@ canvas.addEventListener('click', goToSer
// initialize settings based on request parameters
var main = document.getElementById('main');
var useCircles = true;
-var useIngest = true;
setShape(document.getElementById('shape'));
setSize(document.getElementById('size'));
setMotion(document.getElementById('motion'));
@@ -83,53 +87,55 @@ function handleNewData() {
xmlReturned = true;
return;
}
- var loadinfo = xmlhttp.responseXML.getElementsByTagName('osload');
- var ingestinfo = xmlhttp.responseXML.getElementsByTagName('ingest');
- var queryinfo = xmlhttp.responseXML.getElementsByTagName('query');
+ var statinfo = new Array(statName.length);
+ for (j=0; j < statName.length; j++)
+ statinfo[j] = xmlhttp.responseXML.getElementsByTagName(statName[j]);
var deadinfo = xmlhttp.responseXML.getElementsByTagName('deadTabletServer');
+ var badinfo = xmlhttp.responseXML.getElementsByTagName('badTabletServer');
var idinfo = xmlhttp.responseXML.getElementsByTagName('server');
- for (i=0; i < loadinfo.length; i++) {
- var load = parseFloat(loadinfo[i].childNodes[0].nodeValue);
- var activity;
- if (useIngest)
- activity = parseFloat(ingestinfo[i].childNodes[0].nodeValue);
- else
- activity = parseFloat(queryinfo[i].childNodes[0].nodeValue);
+ var statValues = new Array(3);
+ for (i=0; i < idinfo.length; i++) {
var info = idinfo[i].attributes[0].nodeValue;
- var extra = '<br>osload: ' + Math.round(load*100)/100 + ', ' + (useIngest?'ingest':'query') + ': ' + Math.round(activity);
- newColor = load;
- if (activity < 10) {
- newSkip = -1;
- } else {
- newSkip = Math.ceil(Math.pow(1.5,maxThroughput - Math.log(activity)/Math.log(10)));
+ var extra = '<br>';
+ for (j=0; j < statinfo.length; j++) {
+ statValues[j] = Math.floor(significance[j]*parseFloat(statinfo[j][i].childNodes[0].nodeValue))/significance[j];
+ if (maxStatValue[j] < statValues[j])
+ maxStatValue[j] = statValues[j];
+ extra = extra + ' ' + statName[j] + ': ' + statValues[j];
}
- setDotInfo(newColor,newSkip,info,extra,i);
+ setDotInfo(statValues[colorStatType],statValues[speedStatType],info,extra,i);
}
- for (i=loadinfo.length,j=0; j < deadinfo.length; i++,j++) {
+ for (i=idinfo.length,j=0; j < deadinfo.length; i++,j++) {
setDotInfo(-1,-1,deadinfo[j].attributes[0].nodeValue,'',i);
}
- if (numDots != loadinfo.length + deadinfo.length)
+ for (i=idinfo.length+deadinfo.length,j=0; j < badinfo.length; i++,j++) {
+ setDotInfo(-1,-1,badinfo[j].attributes[0].nodeValue,'',i);
+ }
+ if (numDots != idinfo.length + deadinfo.length + badinfo.length)
drawGrid();
- numDots = loadinfo.length + deadinfo.length;
+ numDots = idinfo.length + deadinfo.length + badinfo.length;
xmlReturned = true;
}
-function setDotInfo(color,resizeModulus,id,extra,index) {
+function setDotInfo(color,rate,id,extra,index) {
if (maxObservedColor < color) {
maxObservedColor = color;
}
+ if (maxObservedSpeed < rate) {
+ maxObservedSpeed = rate;
+ }
if (index >= colors.length) {
colors.push(color);
- resizeFrameModulus.push(resizeModulus);
+ speed.push(rate);
ids.push(id);
extras.push(extra);
dotSize.push(maxDotRadius);
dotSizeGrowing.push(false);
} else {
colors[index] = color;
- resizeFrameModulus[index] = resizeModulus;
+ speed[index] = rate;
ids[index] = id;
extras[index] = extra;
// keep existing size and direction
@@ -151,25 +157,35 @@ function drawDots(){
var x;
var y;
for (i=0; i < numDots; i++) {
- if (resizeFrameModulus[i]<=0 || dotSize[i] > maxDotRadius) {
+ if (Math.floor(dotSize[i]) > maxDotRadius) {
+ // check for resize by the user
+ dotSize[i] = maxDotRadius;
+ } else if (speed[i]<=0) {
// if not changing size, increase to max radius
- // also check for resize by the user
if (dotSize[i] < maxDotRadius)
dotSize[i] = dotSize[i] + 1;
- else
+ if (dotSize[i] > maxDotRadius)
dotSize[i] = maxDotRadius;
- } else if (resizeFrameModulus[i] > 0 && frame % resizeFrameModulus[i] == 0) {
- if (dotSize[i] >= maxDotRadius) {dotSizeGrowing[i] = false};
- if (dotSize[i] <= minDotRadius) {dotSizeGrowing[i] = true};
- if (dotSizeGrowing[i]) {dotSize[i] = dotSize[i] + 1;}
- else {dotSize[i] = dotSize[i] - 1;}
+ } else {
+ if (dotSizeGrowing[i]) {
+ dotSize[i] = dotSize[i] + speed[i]/maxObservedSpeed;
+ if (dotSize[i] + speed[i]/maxObservedSpeed > maxDotRadius) {
+ dotSizeGrowing[i] = false;
+ }
+ }
+ else {
+ dotSize[i] = dotSize[i] - speed[i]/maxObservedSpeed;
+ if (dotSize[i] - speed[i]/maxObservedSpeed < minDotRadius) {
+ dotSizeGrowing[i] = true;
+ }
+ }
}
x = i % width;
y = Math.floor(i / width);
if (colors[i]==-1)
strokeDot(x,y,maxDotRadius-1,deadColor);
else
- drawDot(x,y,dotSize[i],getColor(colors[i]));
+ drawDot(x,y,Math.floor(dotSize[i]),getColor(colors[i]));
}
if (mousedDot < numDots)
document.getElementById('vishoverinfo').innerHTML=ids[mousedDot]+extras[mousedDot];
@@ -178,9 +194,9 @@ function drawDots(){
// fill in a few grey dots
function drawGrid() {
context.clearRect(0, 0, canvas.width, canvas.height);
- for (i=0; i < 32; i++) {
- for (j=0; j < 32; j++) {
- drawDot(i,j,maxDotRadius,nullColor);
+ for (i=0,k=0; i < canvas.width; i+=dotSpacing*2,k++) {
+ for (j=0,l=0; j < canvas.height; j+=dotSpacing*2,l++) {
+ drawDot(k,l,maxDotRadius,nullColor);
}
}
}
@@ -271,6 +287,10 @@ function setSize(obj) {
dotSpacing = 20;
minDotRadius = 5;
break;
+ case 3:
+ dotSpacing = 40;
+ minDotRadius = 7;
+ break;
default:
dotSpacing = 10;
minDotRadius = 3;
@@ -282,16 +302,16 @@ function setSize(obj) {
// callback for motion selection
function setMotion(obj) {
- switch (obj.selectedIndex) {
- case 0:
- useIngest = true;
- break;
- case 1:
- useIngest = false;
- break;
- default:
- useIngest = true;
- }
+ speedStatType = obj.selectedIndex;
+ maxObservedSpeed = maxStatValue[speedStatType];
+ drawGrid();
+ setState();
+}
+
+// callback for color selection
+function setColor(obj) {
+ colorStatType = obj.selectedIndex;
+ maxObservedColor = maxStatValue[colorStatType];
drawGrid();
setState();
}
@@ -328,8 +348,8 @@ function showId(e) {
}
function setState() {
- var url = visurl+'?shape='+(useCircles?'circles':'squares')+'&size='+(dotSpacing*2)+'&motion='+(useIngest?'ingest':'query');
- window.history.replaceState(window.history.state,'Tablet Server Status Visualization',url);
+ var url = visurl+'?shape='+(useCircles?'circles':'squares')+'&size='+(dotSpacing*2)+'&motion='+statName[speedStatType]+'&color='+statName[colorStatType];
+ window.history.replaceState(window.history.state,'Server Activity',url);
}
// go to server page on click
Modified: incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java (original)
+++ incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java Wed Feb 29 21:05:58 2012
@@ -132,7 +132,7 @@ public class TestMergeState {
// do the state check
MergeStats stats = scan(state, metaDataStateStore);
- MergeState newState = stats.nextMergeState();
+ MergeState newState = stats.nextMergeState(connector, state);
Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);
// unassign the tablets
@@ -141,12 +141,9 @@ public class TestMergeState {
deleter.setRanges(Collections.singletonList(new Range()));
deleter.delete();
- // now we should be ready to merge
+ // now we should be ready to merge but, we have an inconsistent !METADATA table
stats = scan(state, metaDataStateStore);
- Assert.assertEquals(MergeState.MERGING, stats.nextMergeState());
-
- // but, we have an inconsistent !METADATA table, so double check
- Assert.assertFalse(stats.verifyMergeConsistency(connector, state));
+ Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
// finish the split
KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
@@ -157,7 +154,7 @@ public class TestMergeState {
// onos... there's a new tablet online
stats = scan(state, metaDataStateStore);
- Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState());
+ Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(connector, state));
// chop it
m = tablet.getPrevRowUpdateMutation();
@@ -165,7 +162,7 @@ public class TestMergeState {
update(connector, m);
stats = scan(state, metaDataStateStore);
- Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState());
+ Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
// take it offline
m = tablet.getPrevRowUpdateMutation();
@@ -174,10 +171,7 @@ public class TestMergeState {
// now we can split
stats = scan(state, metaDataStateStore);
- Assert.assertEquals(MergeState.MERGING, stats.nextMergeState());
-
- // and we have consistent !METADATA table
- Assert.assertTrue(stats.verifyMergeConsistency(connector, state));
+ Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));
}