You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/01/27 20:56:31 UTC
svn commit: r1236863 - in /incubator/accumulo/trunk: ./ src/core/
src/server/ src/server/src/main/java/org/apache/accumulo/server/master/
src/server/src/main/java/org/apache/accumulo/server/master/state/
src/server/src/main/java/org/apache/accumulo/ser...
Author: ecn
Date: Fri Jan 27 19:56:30 2012
New Revision: 1236863
URL: http://svn.apache.org/viewvc?rev=1236863&view=rev
Log:
ACCUMULO-315 merge to trunk
Added:
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
- copied unchanged from r1236862, incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
- copied unchanged from r1236862, incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/server/ (props changed)
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/RootTabletStateStore.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/FlushTablets.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Module.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/AddressUtil.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 27 19:56:30 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1236425
+/incubator/accumulo/branches/1.4:1201902-1236862
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 27 19:56:30 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/core:1209938
/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1236425
+/incubator/accumulo/branches/1.3.5rc/src/core:1209938
+/incubator/accumulo/branches/1.4/src/core:1201902-1236862
Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 27 19:56:30 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/server:1209938
/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
-/incubator/accumulo/branches/1.4/src/server:1201902-1236425
+/incubator/accumulo/branches/1.3.5rc/src/server:1209938
+/incubator/accumulo/branches/1.4/src/server:1201902-1236862
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java Fri Jan 27 19:56:30 2012
@@ -110,6 +110,7 @@ import org.apache.accumulo.server.master
import org.apache.accumulo.server.master.state.DistributedStoreException;
import org.apache.accumulo.server.master.state.MergeInfo;
import org.apache.accumulo.server.master.state.MergeState;
+import org.apache.accumulo.server.master.state.MergeStats;
import org.apache.accumulo.server.master.state.MetaDataStateStore;
import org.apache.accumulo.server.master.state.RootTabletStateStore;
import org.apache.accumulo.server.master.state.TServerInstance;
@@ -196,52 +197,6 @@ public class Master implements LiveTServ
final private static int MAX_TSERVER_WORK_CHUNK = 5000;
final private static int MAX_BAD_STATUS_COUNT = 3;
- static class MergeStats {
- MergeInfo info;
- int hosted = 0;
- int unassigned = 0;
- int chopped = 0;
- int needsToBeChopped = 0;
- int total = 0;
- boolean lowerSplit = false;
- boolean upperSplit = false;
-
- public MergeStats(MergeInfo info) {
- this.info = info;
- if (info.getState().equals(MergeState.NONE))
- return;
- if (info.getRange().getEndRow() == null)
- upperSplit = true;
- if (info.getRange().getPrevEndRow() == null)
- lowerSplit = true;
- }
-
- void update(KeyExtent ke, TabletState state, boolean chopped) {
- if (info.getState().equals(MergeState.NONE))
- return;
- if (!upperSplit && info.getRange().getEndRow().equals(ke.getPrevEndRow())) {
- log.info("Upper split found");
- upperSplit = true;
- }
- if (!lowerSplit && info.getRange().getPrevEndRow().equals(ke.getEndRow())) {
- log.info("Lower split found");
- lowerSplit = true;
- }
- if (!info.overlaps(ke))
- return;
- if (info.needsToBeChopped(ke)) {
- this.needsToBeChopped++;
- if (chopped)
- this.chopped++;
- }
- this.total++;
- if (state.equals(TabletState.HOSTED))
- this.hosted++;
- if (state.equals(TabletState.UNASSIGNED))
- this.unassigned++;
- }
- }
-
final private Instance instance;
final private String hostname;
final private FileSystem fs;
@@ -1366,13 +1321,13 @@ public class Master implements LiveTServ
if (mergeStats == null) {
mergeStatsCache.put(tableId, mergeStats = new MergeStats(getMergeInfo(tableId)));
}
- TabletGoalState goal = getGoalState(tls, mergeStats.info);
+ TabletGoalState goal = getGoalState(tls, mergeStats.getMergeInfo());
TServerInstance server = tls.getServer();
TabletState state = tls.getState(currentTServers.keySet());
stats.update(tableId, state);
mergeStats.update(tls.extent, state, tls.chopped);
- sendChopRequest(mergeStats.info, state, tls);
- sendSplitRequest(mergeStats.info, state, tls);
+ sendChopRequest(mergeStats.getMergeInfo(), state, tls);
+ sendSplitRequest(mergeStats.getMergeInfo(), state, tls);
// Always follow through with assignments
if (state == TabletState.ASSIGNED) {
@@ -1529,84 +1484,31 @@ public class Master implements LiveTServ
private void updateMergeState(Map<Text,MergeStats> mergeStatsCache) {
for (MergeStats stats : mergeStatsCache.values()) {
- MergeState state = stats.info.getState();
try {
- if (state == MergeState.STARTED) {
- setMergeState(stats.info, state = MergeState.SPLITTING);
- }
- if (state == MergeState.SPLITTING) {
- log.info(stats.hosted + " are hosted, total " + stats.total);
- if (!stats.info.isDelete() && stats.total == 1) {
- log.info("Merge range is already contained in a single tablet");
- setMergeState(stats.info, state = MergeState.COMPLETE);
- } else if (stats.hosted == stats.total) {
- if (stats.info.isDelete()) {
- if (!stats.lowerSplit)
- log.info("Waiting for " + stats.info + " lower split to occur");
- else if (!stats.upperSplit)
- log.info("Waiting for " + stats.info + " upper split to occur");
- else
- setMergeState(stats.info, state = MergeState.WAITING_FOR_CHOPPED);
- } else {
- setMergeState(stats.info, state = MergeState.WAITING_FOR_CHOPPED);
- }
- } else {
- log.info("Waiting for " + stats.hosted + " hosted tablets to be " + stats.total);
- }
- }
- if (state == MergeState.WAITING_FOR_CHOPPED) {
- log.info(stats.chopped + " tablets are chopped");
- if (stats.chopped == stats.needsToBeChopped) {
- setMergeState(stats.info, state = MergeState.WAITING_FOR_OFFLINE);
- } else {
- log.info("Waiting for " + stats.chopped + " chopped tablets to be " + stats.needsToBeChopped);
- }
- }
- if (state == MergeState.WAITING_FOR_OFFLINE) {
- if (stats.chopped != stats.needsToBeChopped) {
- log.warn("Unexpected state: chopped tablets should be " + stats.needsToBeChopped + " was " + stats.chopped + " merge " + stats.info.getRange());
- // Perhaps a split occurred after we chopped, but before we went offline: start over
- setMergeState(stats.info, state = MergeState.SPLITTING);
- } else {
- log.info(stats.chopped + " tablets are chopped, " + stats.unassigned + " are offline");
- if (stats.unassigned == stats.total && stats.chopped == stats.needsToBeChopped) {
- setMergeState(stats.info, state = MergeState.MERGING);
- } else {
- log.info("Waiting for " + stats.unassigned + " unassigned tablets to be " + stats.total);
- }
- }
- }
- if (state == MergeState.MERGING) {
- if (stats.hosted != 0) {
- // Shouldn't happen
- log.error("Unexpected state: hosted tablets should be zero " + stats.hosted + " merge " + stats.info.getRange());
- }
- if (stats.unassigned != stats.total) {
- // Shouldn't happen
- log.error("Unexpected state: unassigned tablets should be " + stats.total + " was " + stats.unassigned + " merge " + stats.info.getRange());
- }
- log.info(stats.unassigned + " tablets are unassigned");
- if (stats.hosted == 0 && stats.unassigned == stats.total) {
- try {
- if (stats.info.isDelete())
- deleteTablets(stats.info);
- else
- mergeMetadataRecords(stats.info);
- setMergeState(stats.info, state = MergeState.COMPLETE);
- } catch (Exception ex) {
- log.error("Unable merge metadata table records", ex);
+ MergeState update = stats.nextMergeState();
+ if (update != stats.getMergeInfo().getState()) {
+ if (update == MergeState.MERGING) {
+ if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
+ try {
+ if (stats.getMergeInfo().isDelete())
+ deleteTablets(stats.getMergeInfo());
+ else
+ mergeMetadataRecords(stats.getMergeInfo());
+ setMergeState(stats.getMergeInfo(), MergeState.COMPLETE);
+ update = MergeState.NONE;
+ } catch (Exception ex) {
+ log.error("Unable merge metadata table records", ex);
+ }
}
}
- }
- if (state == MergeState.COMPLETE) {
- setMergeState(stats.info, MergeState.NONE);
+ setMergeState(stats.getMergeInfo(), update);
}
} catch (Exception ex) {
- log.error("Unable to update merge state for merge " + stats.info.getRange(), ex);
+ log.error("Unable to update merge state for merge " + stats.getMergeInfo().getRange(), ex);
}
}
}
-
+
private void deleteTablets(MergeInfo info) throws AccumuloException {
KeyExtent range = info.getRange();
log.debug("Deleting tablets for " + range);
@@ -1858,6 +1760,7 @@ public class Master implements LiveTServ
}
+
private class MigrationCleanupThread extends Daemon {
public void run() {
@@ -2113,7 +2016,9 @@ public class Master implements LiveTServ
tserverSet.startListeningForTabletServerChanges();
- final TabletStateStore stores[] = {new ZooTabletStateStore(new ZooStore(zroot)), new RootTabletStateStore(this), new MetaDataStateStore(this)};
+ AuthInfo systemAuths = SecurityConstants.getSystemCredentials();
+ final TabletStateStore stores[] = {new ZooTabletStateStore(new ZooStore(zroot)), new RootTabletStateStore(instance, systemAuths, this),
+ new MetaDataStateStore(instance, systemAuths, this)};
for (int i = 0; i < stores.length; i++) {
watchers.add(new TabletGroupWatcher(stores[i]));
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java Fri Jan 27 19:56:30 2012
@@ -21,9 +21,11 @@ import java.util.Iterator;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.impl.BatchWriterImpl;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.hadoop.io.Text;
@@ -35,15 +37,23 @@ public class MetaDataStateStore extends
private static final int LATENCY = 1000;
private static final int MAX_MEMORY = 200 * 1024 * 1024;
+ final protected Instance instance;
final protected CurrentState state;
+ final protected AuthInfo auths;
- public MetaDataStateStore(CurrentState state) {
+ public MetaDataStateStore(Instance instance, AuthInfo auths, CurrentState state) {
+ this.instance = instance;
this.state = state;
+ this.auths = auths;
}
+ public MetaDataStateStore() {
+ this(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), null);
+ }
+
@Override
public Iterator<TabletLocationState> iterator() {
- return new MetaDataTableScanner(Constants.NON_ROOT_METADATA_KEYSPACE, state);
+ return new MetaDataTableScanner(instance, auths, Constants.NON_ROOT_METADATA_KEYSPACE, state);
}
@Override
@@ -69,9 +79,14 @@ public class MetaDataStateStore extends
}
BatchWriter createBatchWriter() {
- BatchWriter writer = new BatchWriterImpl(HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID, MAX_MEMORY,
- LATENCY, THREADS);
- return writer;
+ try {
+ return instance.getConnector(auths).createBatchWriter(Constants.METADATA_TABLE_NAME, MAX_MEMORY, LATENCY, THREADS);
+ } catch (TableNotFoundException e) {
+ // ya, I don't think so
+ throw new RuntimeException(e);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
@Override
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java Fri Jan 27 19:56:30 2012
@@ -32,15 +32,16 @@ import java.util.SortedMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.accumulo.core.util.ColumnFQ;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -50,33 +51,37 @@ public class MetaDataTableScanner implem
BatchScanner mdScanner;
Iterator<Entry<Key,Value>> iter;
- public MetaDataTableScanner(Range range, CurrentState state) {
+ public MetaDataTableScanner(Instance instance, AuthInfo auths, Range range, CurrentState state) {
// scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
try {
- Connector connector = HdfsZooInstance.getInstance().getConnector(SecurityConstants.getSystemCredentials());
+ Connector connector = instance.getConnector(auths);
mdScanner = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
- ColumnFQ.fetch(mdScanner, Constants.METADATA_PREV_ROW_COLUMN);
- mdScanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
- mdScanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
- mdScanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
- mdScanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
+ configureScanner(mdScanner, state);
mdScanner.setRanges(Collections.singletonList(range));
- mdScanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
- IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
- if (state != null) {
- TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
- TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
- TabletStateChangeIterator.setMerges(tabletChange, state.merges());
- }
- mdScanner.addScanIterator(tabletChange);
iter = mdScanner.iterator();
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
- public MetaDataTableScanner(Range range) {
- this(range, null);
+ public MetaDataTableScanner(Instance instance, AuthInfo auths, Range range) {
+ this(instance, auths, range, null);
}
public void close() {
@@ -115,7 +120,8 @@ public class MetaDataTableScanner implem
}
}
- public static TabletLocationState createTabletLocationState(SortedMap<Key,Value> decodedRow) {
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
KeyExtent extent = null;
TServerInstance future = null;
TServerInstance current = null;
@@ -152,10 +158,9 @@ public class MetaDataTableScanner implem
}
private TabletLocationState fetch() {
- Entry<Key,Value> entry = iter.next();
try {
- final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(entry.getKey(), entry.getValue());
- return createTabletLocationState(decodedRow);
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
} catch (IOException ex) {
throw new RuntimeException(ex);
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/RootTabletStateStore.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/RootTabletStateStore.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/RootTabletStateStore.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/RootTabletStateStore.java Fri Jan 27 19:56:30 2012
@@ -19,21 +19,23 @@ package org.apache.accumulo.server.maste
import java.util.Iterator;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.thrift.AuthInfo;
import org.apache.hadoop.io.Text;
public class RootTabletStateStore extends MetaDataStateStore {
- public RootTabletStateStore(CurrentState state) {
- super(state);
+ public RootTabletStateStore(Instance instance, AuthInfo auths, CurrentState state) {
+ super(instance, auths, state);
}
@Override
public Iterator<TabletLocationState> iterator() {
Range range = new Range(Constants.ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text(Constants.METADATA_TABLE_ID), null),
true);
- return new MetaDataTableScanner(range, state);
+ return new MetaDataTableScanner(instance, auths, range, state);
}
@Override
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateChangeIterator.java Fri Jan 27 19:56:30 2012
@@ -118,7 +118,7 @@ public class TabletStateChangeIterator e
return;
SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
- TabletLocationState tls = MetaDataTableScanner.createTabletLocationState(decodedRow);
+ TabletLocationState tls = MetaDataTableScanner.createTabletLocationState(k, v);
// we always want data about merges
MergeInfo merge = merges.get(tls.extent.getTableId());
if (merge != null && merge.getRange() != null && merge.getRange().overlaps(tls.extent)) {
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java Fri Jan 27 19:56:30 2012
@@ -77,7 +77,7 @@ public abstract class TabletStateStore i
if (tls.extent.equals(Constants.ROOT_TABLET_EXTENT)) {
store = new ZooTabletStateStore();
} else {
- store = new MetaDataStateStore(null);
+ store = new MetaDataStateStore();
}
store.unassign(Collections.singletonList(tls));
}
@@ -87,7 +87,7 @@ public abstract class TabletStateStore i
if (assignment.tablet.equals(Constants.ROOT_TABLET_EXTENT)) {
store = new ZooTabletStateStore();
} else {
- store = new MetaDataStateStore(null);
+ store = new MetaDataStateStore();
}
store.setLocations(Collections.singletonList(assignment));
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java Fri Jan 27 19:56:30 2012
@@ -89,7 +89,7 @@ class CleanUp extends MasterRepo {
boolean done = true;
Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
- MetaDataTableScanner metaDataTableScanner = new MetaDataTableScanner(tableRange, null);
+ MetaDataTableScanner metaDataTableScanner = new MetaDataTableScanner(environment.getInstance(), SecurityConstants.getSystemCredentials(), tableRange, null);
try {
while (metaDataTableScanner.hasNext()) {
TabletLocationState locationState = metaDataTableScanner.next();
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/FlushTablets.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/FlushTablets.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/FlushTablets.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/tserverOps/FlushTablets.java Fri Jan 27 19:56:30 2012
@@ -19,9 +19,9 @@ package org.apache.accumulo.server.maste
import java.util.Collection;
import org.apache.accumulo.server.fate.Repo;
-import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.EventCoordinator.Listener;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
+import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.state.DistributedStoreException;
import org.apache.accumulo.server.master.state.MetaDataStateStore;
import org.apache.accumulo.server.master.state.TabletLocationState;
@@ -58,7 +58,7 @@ public class FlushTablets extends Master
} catch (DistributedStoreException e) {
log.warn("Unable to open ZooTabletStateStore, will retry", e);
}
- MetaDataStateStore theRest = new MetaDataStateStore(null);
+ MetaDataStateStore theRest = new MetaDataStateStore();
for (TabletStateStore store : new TabletStateStore[] {zooTabletStateStore, theRest}) {
if (store != null) {
for (TabletLocationState tabletState : store) {
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java Fri Jan 27 19:56:30 2012
@@ -27,6 +27,7 @@ import java.util.TreeSet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Range;
@@ -44,6 +45,7 @@ import org.apache.accumulo.server.monito
import org.apache.accumulo.server.monitor.util.celltypes.NumberType;
import org.apache.accumulo.server.monitor.util.celltypes.TableLinkType;
import org.apache.accumulo.server.monitor.util.celltypes.TableStateType;
+import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.hadoop.io.Text;
public class TablesServlet extends BasicServlet {
@@ -138,8 +140,10 @@ public class TablesServlet extends Basic
private void doTableDetails(HttpServletRequest req, StringBuilder sb, Map<String,String> tidToNameMap, String tableId) {
String displayName = Tables.getPrintableTableNameFromId(tidToNameMap, tableId);
-
- MetaDataTableScanner scanner = new MetaDataTableScanner(new Range(KeyExtent.getMetadataEntry(new Text(tableId), new Text()), KeyExtent.getMetadataEntry(
+ Instance instance = HdfsZooInstance.getInstance();
+ MetaDataTableScanner scanner = new MetaDataTableScanner(instance, SecurityConstants.getSystemCredentials(), new Range(KeyExtent.getMetadataEntry(new Text(
+ tableId), new Text()),
+ KeyExtent.getMetadataEntry(
new Text(tableId), null)));
TreeSet<String> locs = new TreeSet<String>();
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java Fri Jan 27 19:56:30 2012
@@ -54,6 +54,7 @@ import org.apache.accumulo.server.master
import org.apache.accumulo.server.master.state.MetaDataTableScanner;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.accumulo.server.util.TServerUtils;
import org.apache.accumulo.server.zookeeper.TransactionWatcher;
import org.apache.hadoop.io.Text;
@@ -207,7 +208,7 @@ public class NullTserver {
// read the locations for the table
Range tableRange = new KeyExtent(new Text(tableId), null, null).toMetadataRange();
- MetaDataTableScanner s = new MetaDataTableScanner(tableRange);
+ MetaDataTableScanner s = new MetaDataTableScanner(zki, SecurityConstants.getSystemCredentials(), tableRange);
long randomSessionID = port;
TServerInstance instance = new TServerInstance(addr, randomSessionID);
List<Assignment> assignments = new ArrayList<Assignment>();
@@ -217,7 +218,7 @@ public class NullTserver {
}
s.close();
// point them to this server
- MetaDataStateStore store = new MetaDataStateStore(null);
+ MetaDataStateStore store = new MetaDataStateStore();
store.setLocations(assignments);
while (true) {
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Module.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Module.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Module.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/Module.java Fri Jan 27 19:56:30 2012
@@ -20,6 +20,7 @@ import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
@@ -240,16 +241,21 @@ public class Module extends Node {
nextNodeId = ((Alias) nextNode).getTargetId();
nextNode = ((Alias) nextNode).get();
}
+ Properties nodeProps = getProps(nextNodeId);
try {
test = false;
if (nextNode instanceof Test) {
startTimer(nextNode.toString());
test = true;
}
- nextNode.visit(state, getProps(nextNodeId));
+ nextNode.visit(state, nodeProps);
if (test)
stopTimer(nextNode.toString());
} catch (Exception e) {
+ log.debug("Properties for node: " + nextNodeId);
+ for (Entry<Object,Object> entry : nodeProps.entrySet()) {
+ log.debug(" " + entry.getKey() + ": " + entry.getValue());
+ }
throw new Exception("Error running node " + nextNodeId, e);
}
state.visitedNode();
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java Fri Jan 27 19:56:30 2012
@@ -92,10 +92,10 @@ public class TableOp extends Test {
Key k = entry.getKey();
seen++;
if (!auths.contains(k.getColumnVisibilityData()))
- throw new AccumuloException("Got data I should not be capable of seeing");
+ throw new AccumuloException("Got data I should not be capable of seeing: " + k + " table " + tableName);
}
if (!canRead)
- throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami());
+ throw new AccumuloException("Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName);
for (Entry<String,Integer> entry : SecurityHelper.getAuthsMap(state).entrySet()) {
if (auths.contains(entry.getKey().getBytes()))
seen = seen - entry.getValue();
@@ -104,12 +104,12 @@ public class TableOp extends Test {
throw new AccumuloException("Got mismatched amounts of data");
} catch (TableNotFoundException tnfe) {
if (tableExists)
- throw new AccumuloException("Accumulo and test suite out of sync", tnfe);
+ throw new AccumuloException("Accumulo and test suite out of sync: table " + tableName, tnfe);
return;
} catch (AccumuloSecurityException ae) {
if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (canRead)
- throw new AccumuloException("Table read permission out of sync with Accumulo", ae);
+ throw new AccumuloException("Table read permission out of sync with Accumulo: table " + tableName, ae);
else
return;
}
@@ -118,7 +118,7 @@ public class TableOp extends Test {
if (re.getCause() instanceof AccumuloSecurityException
&& ((AccumuloSecurityException) re.getCause()).getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (canRead)
- throw new AccumuloException("Table read permission out of sync with Accumulo", re.getCause());
+ throw new AccumuloException("Table read permission out of sync with Accumulo: table " + tableName, re.getCause());
else
return;
}
@@ -137,7 +137,7 @@ public class TableOp extends Test {
writer = conn.createBatchWriter(tableName, 9000l, 0l, 1);
} catch (TableNotFoundException tnfe) {
if (tableExists)
- throw new AccumuloException("Table didn't exist when it should have");
+ throw new AccumuloException("Table didn't exist when it should have: " + tableName);
return;
}
boolean works = true;
@@ -171,12 +171,12 @@ public class TableOp extends Test {
conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true);
} catch (TableNotFoundException tnfe) {
if (tableExists)
- throw new AccumuloException("Table didn't exist when it should have");
+ throw new AccumuloException("Table didn't exist when it should have: " + tableName);
return;
} catch (AccumuloSecurityException ae) {
if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (hasPerm)
- throw new AccumuloException("Bulk Import failed when it should have worked.");
+ throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName);
return;
}
throw new AccumuloException("Unexpected exception!", ae);
@@ -185,7 +185,7 @@ public class TableOp extends Test {
SecurityHelper.increaseAuthMap(state, s, 1);
if (!hasPerm)
- throw new AccumuloException("Bulk Import succeeded when it should have failed.");
+ throw new AccumuloException("Bulk Import succeeded when it should have failed: " + dir + " table " + tableName);
break;
case ALTER_TABLE:
AlterTable.renameTable(conn, state, tableName, tableName + "plus", hasPerm, tableExists);
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/AddressUtil.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/AddressUtil.java?rev=1236863&r1=1236862&r2=1236863&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/AddressUtil.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/util/AddressUtil.java Fri Jan 27 19:56:30 2012
@@ -23,7 +23,7 @@ import org.apache.accumulo.server.conf.S
public class AddressUtil {
static public InetSocketAddress parseAddress(String address, Property portDefaultProperty) {
- final int dfaultPort = ServerConfiguration.getSystemConfiguration().getPort(Property.TSERV_CLIENTPORT);
+ final int dfaultPort = ServerConfiguration.getDefaultConfiguration().getPort(Property.TSERV_CLIENTPORT);
return org.apache.accumulo.core.util.AddressUtil.parseAddress(address, dfaultPort);
}