You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/02/28 01:38:14 UTC
svn commit: r1451062 [2/2] - in /accumulo/branches/ACCUMULO-259-polishing:
./ core/ core/src/main/java/org/apache/accumulo/core/cli/
core/src/main/java/org/apache/accumulo/core/client/admin/
core/src/main/java/org/apache/accumulo/core/client/impl/thrif...
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Feb 28 00:38:13 2013
@@ -81,6 +81,7 @@ import org.apache.accumulo.core.security
import org.apache.accumulo.core.security.thrift.SecurityErrorCode;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Daemon;
@@ -101,7 +102,7 @@ import org.apache.accumulo.server.conf.S
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
import org.apache.accumulo.server.master.balancer.TabletBalancer;
-import org.apache.accumulo.server.master.recovery.RecoverLease;
+import org.apache.accumulo.server.master.recovery.RecoveryManager;
import org.apache.accumulo.server.master.state.Assignment;
import org.apache.accumulo.server.master.state.CurrentState;
import org.apache.accumulo.server.master.state.DeadServerList;
@@ -157,7 +158,6 @@ import org.apache.accumulo.start.classlo
import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
@@ -202,6 +202,7 @@ public class Master implements LiveTServ
final private SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,TServerInstance>());
final private EventCoordinator nextEvent = new EventCoordinator();
final private Object mergeLock = new Object();
+ private RecoveryManager recoveryManager = null;
private ZooLock masterLock = null;
private TServer clientService = null;
@@ -227,15 +228,14 @@ public class Master implements LiveTServ
static final boolean X = true;
static final boolean _ = false;
static final boolean transitionOK[][] = {
- // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
- /* INITIAL */ {X, X, _, _, _, _, X},
- /* HAVE_LOCK */ {_, X, X, X, _, _, X},
- /* SAFE_MODE */ {_, _, X, X, X, _, X},
- /* NORMAL */ {_, _, X, X, X, _, X},
- /* UNLOAD_METADATA_TABLETS */ {_, _, X, X, X, X, X},
- /* UNLOAD_ROOT_TABLET */ {_, _, _, _, _, X, X},
- /* STOP */ {_, _, _, _, _, _, X}
- };
+ // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
+ /* INITIAL */{X, X, _, _, _, _, X},
+ /* HAVE_LOCK */{_, X, X, X, _, _, X},
+ /* SAFE_MODE */{_, _, X, X, X, _, X},
+ /* NORMAL */{_, _, X, X, X, _, X},
+ /* UNLOAD_METADATA_TABLETS */{_, _, X, X, X, X, X},
+ /* UNLOAD_ROOT_TABLET */{_, _, _, X, _, X, X},
+ /* STOP */{_, _, _, _, _, _, X}};
synchronized private void setMasterState(MasterState newState) {
if (state.equals(newState))
@@ -275,11 +275,11 @@ public class Master implements LiveTServ
log.info("Upgrading zookeeper");
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
-
+
zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.SKIP);
-
+
for (String id : Tables.getIdToNameMap(instance).keySet()) {
-
+
zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, "0".getBytes(),
NodeExistsPolicy.SKIP);
}
@@ -289,7 +289,7 @@ public class Master implements LiveTServ
}
}
}
-
+
private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
private final ServerConfiguration serverConfig;
@@ -710,7 +710,8 @@ public class Master implements LiveTServ
@Override
public void reportSplitExtent(TInfo info, TCredentials credentials, String serverName, TabletSplit split) throws TException {
- if (migrations.remove(new KeyExtent(split.oldTablet)) != null) {
+ KeyExtent oldTablet = new KeyExtent(split.oldTablet);
+ if (migrations.remove(oldTablet) != null) {
log.info("Canceled migration of " + split.oldTablet);
}
for (TServerInstance instance : tserverSet.getCurrentServers()) {
@@ -1341,7 +1342,7 @@ public class Master implements LiveTServ
if (goal == TabletGoalState.HOSTED) {
if (state != TabletState.HOSTED && !tls.walogs.isEmpty()) {
- if (recoverLogs(tls.extent, tls.walogs))
+ if (recoveryManager.recoverLogs(tls.extent, tls.walogs))
continue;
}
switch (state) {
@@ -1469,6 +1470,8 @@ public class Master implements LiveTServ
} else {
log.warn("Not connected to server " + tls.current);
}
+ } catch (NotServingTabletException e) {
+ log.debug("Error asking tablet server to split a tablet: " + e);
} catch (Exception e) {
log.warn("Error asking tablet server to split a tablet: " + e);
}
@@ -1858,13 +1861,7 @@ public class Master implements LiveTServ
try {
switch (getMasterGoalState()) {
case NORMAL:
- switch (getMasterState()) {
- case HAVE_LOCK:
- case SAFE_MODE:
- setMasterState(MasterState.NORMAL);
- default:
- break;
- }
+ setMasterState(MasterState.NORMAL);
break;
case SAFE_MODE:
if (getMasterState() == MasterState.NORMAL) {
@@ -2002,7 +1999,10 @@ public class Master implements LiveTServ
String oldName = t.getName();
try {
t.setName("Getting status from " + server);
- TabletServerStatus status = tserverSet.getConnection(server).getTableMap();
+ TServerConnection connection = tserverSet.getConnection(server);
+ if (connection == null)
+ throw new IOException("No connection to " + server);
+ TabletServerStatus status = connection.getTableMap();
// TODO maybe remove from bad servers
result.put(server, status);
} finally {
@@ -2034,39 +2034,13 @@ public class Master implements LiveTServ
return result;
}
- public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>> walogs) throws IOException {
- boolean recoveryNeeded = false;
- for (Collection<String> logs : walogs) {
- for (String log : logs) {
- String parts[] = log.split("/");
- String host = parts[0];
- String filename = parts[1];
- if (fs.exists(new Path(Constants.getRecoveryDir(getSystemConfiguration()) + "/" + filename + "/finished"))) {
- recoveriesInProgress.remove(filename);
- continue;
- }
- recoveryNeeded = true;
- synchronized (recoveriesInProgress) {
- if (!recoveriesInProgress.contains(filename)) {
- Master.log.info("Starting recovery of " + filename + " created for " + host + ", tablet " + extent + " holds a reference");
- AccumuloConfiguration aconf = getConfiguration().getConfiguration();
- RecoverLease impl = createInstanceFromPropertyName(aconf, Property.MASTER_LEASE_RECOVERY_IMPLEMETATION, RecoverLease.class, new RecoverLease());
- impl.init(host, filename);
- long tid = fate.startTransaction();
- fate.seedTransaction(tid, impl, true);
- recoveriesInProgress.add(filename);
- }
- }
- }
- }
- return recoveryNeeded;
- }
-
public void run() throws IOException, InterruptedException, KeeperException {
final String zroot = ZooUtil.getRoot(instance);
getMasterLock(zroot + Constants.ZMASTER_LOCK);
+ recoveryManager = new RecoveryManager(this);
+
TableManager.getInstance().addObserver(this);
StatusThread statusThread = new StatusThread();
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java Thu Feb 28 00:38:13 2013
@@ -103,7 +103,8 @@ import org.apache.thrift.TException;
*/
public class BulkImport extends MasterRepo {
-
+ public static final String FAILURES_TXT = "failures.txt";
+
private static final long serialVersionUID = 1L;
private static final Logger log = Logger.getLogger(BulkImport.class);
@@ -363,13 +364,13 @@ class CopyFailed extends MasterRepo {
FileSystem fs = master.getFileSystem();
- if (!fs.exists(new Path(error, "failures.txt")))
+ if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
return new CleanUpBulkImport(tableId, source, bulk, error);
HashMap<String,String> failures = new HashMap<String,String>();
HashMap<String,String> loadedFailures = new HashMap<String,String>();
- FSDataInputStream failFile = fs.open(new Path(error, "failures.txt"));
+ FSDataInputStream failFile = fs.open(new Path(error, BulkImport.FAILURES_TXT));
BufferedReader in = new BufferedReader(new InputStreamReader(failFile));
try {
String line = null;
@@ -408,7 +409,7 @@ class CopyFailed extends MasterRepo {
Path orig = new Path(failure);
Path dest = new Path(error, orig.getName());
fs.rename(orig, dest);
- log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": failed");
+ log.debug("tid " + tid + " renamed " + orig + " to " + dest + ": import failed");
}
if (loadedFailures.size() > 0) {
@@ -432,7 +433,7 @@ class CopyFailed extends MasterRepo {
bifCopyQueue.waitUntilDone(workIds);
}
- fs.delete(new Path(error, "failures.txt"), true);
+ fs.delete(new Path(error, BulkImport.FAILURES_TXT), true);
return new CleanUpBulkImport(tableId, source, bulk, error);
}
@@ -539,7 +540,7 @@ class LoadFiles extends MasterRepo {
failures.addAll(fail);
}
} catch (Exception ex) {
- log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex, ex);
+ log.error("rpc failed server:" + server + ", tid:" + tid + " " + ex);
} finally {
ServerClient.close(client);
}
@@ -556,7 +557,7 @@ class LoadFiles extends MasterRepo {
}
}
- FSDataOutputStream failFile = fs.create(new Path(errorDir, "failures.txt"), true);
+ FSDataOutputStream failFile = fs.create(new Path(errorDir, BulkImport.FAILURES_TXT), true);
BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile));
try {
for (String f : filesToLoad) {
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/master/tableOps/TableRangeOp.java Thu Feb 28 00:38:13 2013
@@ -92,7 +92,7 @@ class TableRangeOpWait extends MasterRep
public Repo<Master> call(long tid, Master master) throws Exception {
Text tableIdText = new Text(tableId);
MergeInfo mergeInfo = master.getMergeInfo(tableIdText);
- log.warn("removing merge information " + mergeInfo);
+ log.info("removing merge information " + mergeInfo);
master.clearMergeState(tableIdText);
Utils.unreserveTable(tableId, tid, true);
// We can't add entries to the metadata table if it is offline for this merge.
@@ -159,7 +159,7 @@ public class TableRangeOp extends Master
Text tableIdText = new Text(tableId);
MergeInfo mergeInfo = env.getMergeInfo(tableIdText);
if (mergeInfo.getState() != MergeState.NONE)
- log.warn("removing merge information " + mergeInfo);
+ log.info("removing merge information " + mergeInfo);
env.clearMergeState(tableIdText);
Utils.unreserveTable(tableId, tid, true);
}
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Thu Feb 28 00:38:13 2013
@@ -2353,9 +2353,8 @@ public class TabletServer extends Abstra
log.debug("Failed to unload tablet " + extent + "... it was alread closing or closed : " + e.getMessage());
} else {
log.error("Failed to close tablet " + extent + "... Aborting migration", e);
+ enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent));
}
-
- enqueueMasterMessage(new TabletStatusMessage(TabletLoadState.UNLOAD_ERROR, extent));
return;
}
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java Thu Feb 28 00:38:13 2013
@@ -103,7 +103,7 @@ public class LogSorter {
String formerThreadName = Thread.currentThread().getName();
int part = 0;
try {
-
+
// the following call does not throw an exception if the file/dir does not exist
fs.delete(new Path(destPath), true);
@@ -182,7 +182,7 @@ public class LogSorter {
Thread.currentThread().setName(formerThreadName);
try {
close();
- } catch (IOException e) {
+ } catch (Exception e) {
log.error("Error during cleanup sort/copy " + name, e);
}
synchronized (this) {
Modified: accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/server/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java Thu Feb 28 00:38:13 2013
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.zookeeper;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
@@ -106,16 +107,16 @@ public class DistributedWorkQueue {
log.error("Error received when trying to delete entry in zookeeper " + childPath, e);
}
- try {
- zoo.recursiveDelete(lockPath, NodeMissingPolicy.SKIP);
- } catch (Exception e) {
- log.error("Error received when trying to delete entry in zookeeper " + childPath, e);
- }
-
} catch (Exception e) {
log.warn("Failed to process work " + child, e);
}
+ try {
+ zoo.recursiveDelete(lockPath, NodeMissingPolicy.SKIP);
+ } catch (Exception e) {
+ log.error("Error received when trying to delete entry in zookeeper " + childPath, e);
+ }
+
} finally {
numTask.decrementAndGet();
}
@@ -210,6 +211,12 @@ public class DistributedWorkQueue {
zoo.putPersistentData(path + "/" + workId, data, NodeExistsPolicy.SKIP);
}
+ public List<String> getWorkQueued() throws KeeperException, InterruptedException {
+ ArrayList<String> children = new ArrayList<String>(zoo.getChildren(path));
+ children.remove(LOCKS_NODE);
+ return children;
+ }
+
public void waitUntilDone(Set<String> workIDs) throws KeeperException, InterruptedException {
final String condVar = new String("cond");
Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/TestIngest.java Thu Feb 28 00:38:13 2013
@@ -204,7 +204,6 @@ public class TestIngest {
BatchWriter bw = null;
FileSKVWriter writer = null;
- Connector connector = opts.getConnector();
if (opts.outputFile != null) {
Configuration conf = CachedConfiguration.getInstance();
@@ -213,9 +212,10 @@ public class TestIngest {
AccumuloConfiguration.getDefaultConfiguration());
writer.startDefaultLocalityGroup();
} else {
+ Connector connector = opts.getConnector();
bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
+ connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
}
- connector.securityOperations().changeUserAuthorizations(opts.principal, AUTHS);
Text labBA = new Text(opts.columnVisibility.getExpression());
long startTime = System.currentTimeMillis();
Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java Thu Feb 28 00:38:13 2013
@@ -21,9 +21,7 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;
-import org.apache.accumulo.trace.instrument.Trace;
import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
@@ -33,14 +31,13 @@ import org.apache.accumulo.core.data.Ran
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.trace.DistributedTrace;
-import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.trace.instrument.Trace;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
import com.beust.jcommander.Parameter;
-
public class VerifyIngest {
private static final Logger log = Logger.getLogger(VerifyIngest.class);
@@ -54,7 +51,7 @@ public class VerifyIngest {
}
public static class Opts extends TestIngest.Opts {
- @Parameter(names="-useGet", description="fetches values one at a time, instead of scanning")
+ @Parameter(names = "-useGet", description = "fetches values one at a time, instead of scanning")
boolean useGet = false;
}
@@ -71,19 +68,12 @@ public class VerifyIngest {
Trace.currentTrace().data("cmdLine", Arrays.asList(args).toString());
}
- Connector connector = null;
- while (connector == null) {
- try {
- connector = opts.getConnector();
- } catch (AccumuloException e) {
- log.warn("Could not connect to accumulo; will retry: " + e);
- UtilWaitThread.sleep(1000);
- }
- }
+ Connector connector = opts.getConnector();
byte[][] bytevals = TestIngest.generateValues(opts);
Authorizations labelAuths = new Authorizations("L1", "L2", "G1", "GROUP2");
+ connector.securityOperations().changeUserAuthorizations(opts.principal, labelAuths);
int expectedRow = opts.startRow;
int expectedCol = 0;
Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/functional/BulkSplitOptimizationTest.java Thu Feb 28 00:38:13 2013
@@ -21,6 +21,8 @@ import java.util.List;
import java.util.Map;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.security.tokens.PasswordToken;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.test.CreateRFiles;
@@ -62,8 +64,8 @@ public class BulkSplitOptimizationTest e
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
fs.delete(new Path("/tmp/testmf"), true);
-
- CreateRFiles.main(new String[] { "--output", "tmp/testmf", "--numThreads", "8", "--start", "0", "--end", "100000", "--splits", "99"});
+ AuthenticationToken token = this.getToken();
+ CreateRFiles.main(new String[] {"--output", "tmp/testmf", "--numThreads", "8", "--start", "0", "--end", "100000", "--splits", "99"});
bulkImport(fs, TABLE_NAME, "/tmp/testmf");
@@ -82,7 +84,11 @@ public class BulkSplitOptimizationTest e
checkSplits(TABLE_NAME, 50, 100);
- VerifyIngest.main(new String[] {"--timestamp", "1", "--size", "50", "--random", "56", "--rows", "100000", "--start", "0", "--cols", "1"});
+ String passwd = "";
+ if (token instanceof PasswordToken) {
+ passwd = new String(((PasswordToken) token).getPassword());
+ }
+ VerifyIngest.main(new String[] {"--timestamp", "1", "--size", "50", "--random", "56", "--rows", "100000", "--start", "0", "--cols", "1", "-p", passwd});
// ensure each tablet does not have all map files
checkRFiles(TABLE_NAME, 50, 100, 1, 4);
Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java Thu Feb 28 00:38:13 2013
@@ -110,8 +110,10 @@ public class Config extends Test {
String table = parts[0];
int choice = Integer.parseInt(parts[1]);
Property property = tableSettings[choice].property;
- log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue());
- state.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue());
+ if (state.getConnector().tableOperations().exists(table)) {
+ log.debug("Setting " + property.getKey() + " on " + table + " back to " + property.getDefaultValue());
+ state.getConnector().tableOperations().setProperty(table, property.getKey(), property.getDefaultValue());
+ }
}
state.getMap().remove(LAST_SETTING);
state.getMap().remove(LAST_TABLE_SETTING);
@@ -136,9 +138,9 @@ public class Config extends Test {
// generate a random value
long newValue = random.nextLong(setting.min, setting.max);
- state.getMap().put(LAST_TABLE_SETTING, "" + choice);
+ state.getMap().put(LAST_TABLE_SETTING, table + "," + choice);
log.debug("Setting " + setting.property.getKey() + " on table " + table + " to " + newValue);
- state.getConnector().instanceOperations().setProperty(setting.property.getKey(), table + "," + newValue);
+ state.getConnector().tableOperations().setProperty(table, setting.property.getKey(), "" + newValue);
}
private void changeSetting(RandomData random, State state, Properties props) throws Exception {
Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/SecurityFixture.java Thu Feb 28 00:38:13 2013
@@ -21,6 +21,7 @@ import java.util.Set;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.security.tokens.PasswordToken;
@@ -52,7 +53,7 @@ public class SecurityFixture extends Fix
conn.securityOperations().createLocalUser(systemUserName, sysUserPass);
WalkingSecurity.get(state).setTableName(secTableName);
- state.set("rootUserPass", state.getCredentials().getToken());
+ state.set("rootUserPass", CredentialHelper.extractToken(state.getCredentials()));
WalkingSecurity.get(state).setSysUserName(systemUserName);
WalkingSecurity.get(state).createUser(systemUserName, sysUserPass);
Modified: accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/src/main/java/org/apache/accumulo/test/randomwalk/security/WalkingSecurity.java Thu Feb 28 00:38:13 2013
@@ -279,7 +279,7 @@ public class WalkingSecurity extends Sec
}
public byte[] getUserPassword(String user) {
- Object obj = state.get(getSysUserName() + userPass);
+ Object obj = state.get(user + userPass);
if (obj instanceof PasswordToken) {
return ((PasswordToken) obj).getPassword();
}
@@ -295,7 +295,11 @@ public class WalkingSecurity extends Sec
}
public byte[] getTabPassword() {
- return (byte[]) state.get(getTabUserName() + userPass);
+ Object obj = state.get(getTabUserName() + userPass);
+ if (obj instanceof PasswordToken) {
+ return ((PasswordToken) obj).getPassword();
+ }
+ return null;
}
public boolean userPassTransient(String user) {
Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/TestUtils.py Thu Feb 28 00:38:13 2013
@@ -120,7 +120,7 @@ class TestUtilsMixin:
ld = 'ACCUMULO_LOG_DIR=%s/logs/%s' % (ACCUMULO_HOME, ID)
execcmd = ['ssh', '-q', host, cp, jo, go, ld] + quote(cmd)
log.debug(repr(execcmd))
- return Popen(execcmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, **opts)
+ return Popen(execcmd, stdout=PIPE, stderr=PIPE, **opts)
def shell(self, host, input, **opts):
"""Run accumulo shell with the given input,
@@ -220,7 +220,7 @@ class TestUtilsMixin:
args = ''
if timestamp:
args += "-ts %ld " % int(timestamp)
- args += '--debug -i %s -u %s --size %d --random 56 --rows %d --start %d --cols 1 --createTable ' % (INSTANCE_NAME, ROOT, size, count, start)
+ args += '--debug -i %s -u %s --size %d --random 56 --rows %d --start %d --cols 1 --createTable -p %s' % (INSTANCE_NAME, ROOT, size, count, start, ROOT_PASSWORD)
if colf:
args = '--columnFamily %s ' % colf + args
return self.runClassOn(host, klass, args.split(), **kwargs)
@@ -230,7 +230,7 @@ class TestUtilsMixin:
args = ''
if timestamp:
args += "-ts %ld " % int(timestamp)
- args += '-i %s -u %s --size %d --random 56 -cf %s --rows %d --start %d --cols 1 ' % (INSTANCE_NAME, ROOT, size, colf, count, start)
+ args += '-i %s -u %s --size %d --random 56 -cf %s --rows %d --start %d --cols 1 -p %s' % (INSTANCE_NAME, ROOT, size, colf, count, start, ROOT_PASSWORD)
return self.runClassOn(host, klass, args.split())
def stop_accumulo(self, signal=signal.SIGHUP):
Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/binary.py Thu Feb 28 00:38:13 2013
@@ -36,7 +36,7 @@ class BinaryTest(unittest.TestCase, Test
handle = self.runClassOn(self.masterHost(), testClass,
['-u', 'root',
'--mode', mode, '-t', table,
- '--start', min, '--count', max])
+ '--start', min, '--count', max, '-p', 'secret'])
self.waitForStop(handle, 200)
def tearDown(self):
Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/examples.py Thu Feb 28 00:38:13 2013
@@ -63,7 +63,7 @@ class Examples(TestUtilsMixin, unittest.
self.comment("Testing MaxMutation constraint")
self.ashell('createtable test_ingest\n'
'constraint -a org.apache.accumulo.examples.simple.constraints.MaxMutationSize\n')
- handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000'])
+ handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.test.TestIngest', '-u', ROOT, '--rows', '1', '--start', '0', '--cols', '10000', '-p', ROOT_PASSWORD])
out, err = handle.communicate()
self.failIf(handle.returncode==0)
self.failUnless(err.find("MutationsRejectedException: # constraint violations : 1") >= 0, "Was able to insert a mutation larger than max size")
Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/simple/readwrite.py Thu Feb 28 00:38:13 2013
@@ -20,7 +20,7 @@ import unittest
import time
import sys
-from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE, INSTANCE_NAME
+from TestUtils import TestUtilsMixin, FUZZ, ACCUMULO_HOME, SITE, INSTANCE_NAME, ROOT, ROOT_PASSWORD
log = logging.getLogger('test.auto')
@@ -82,12 +82,12 @@ class MultiTableTest(SunnyDayTest):
def ingest(self, host, count, *args, **kwargs):
klass = 'org.apache.accumulo.test.TestMultiTableIngest'
- args = '--count %d -u root -i %s ' % (count, INSTANCE_NAME)
+ args = '--count %d -u root -i %s -p %s' % (count, INSTANCE_NAME, ROOT_PASSWORD)
return self.runClassOn(host, klass, args.split())
def verify(self, host, count, *args, **kwargs):
klass = 'org.apache.accumulo.test.TestMultiTableIngest'
- args = '--count %d --readonly -u root -i %s ' % (count, INSTANCE_NAME)
+ args = '--count %d --readonly -u root -i %s -p %s' % (count, INSTANCE_NAME, ROOT_PASSWORD)
return self.runClassOn(host, klass, args.split())
Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/auto/stress/batchWrite.py Thu Feb 28 00:38:13 2013
@@ -19,7 +19,7 @@ import logging
import unittest
import time
-from TestUtils import TestUtilsMixin
+from TestUtils import TestUtilsMixin, ROOT_PASSWORD
log = logging.getLogger('test.auto')
@@ -33,7 +33,7 @@ class WriteLots(unittest.TestCase, TestU
def ingest(self, host, start, count, **kwargs):
klass = 'org.apache.accumulo.test.TestIngest'
- args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1' % (count, start)
+ args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1 -p %s' % (count, start, ROOT_PASSWORD)
return self.runClassOn(host, klass, args.split(), **kwargs)
def setUp(self):
Modified: accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl?rev=1451062&r1=1451061&r2=1451062&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl (original)
+++ accumulo/branches/ACCUMULO-259-polishing/test/system/continuous/agitator.pl Thu Feb 28 00:38:13 2013
@@ -19,15 +19,38 @@
use POSIX qw(strftime);
if(scalar(@ARGV) != 4 && scalar(@ARGV) != 2){
- print "Usage : agitator.pl <sleep before kill in minutes> <sleep before tup in minutes> [<min kill> <max kill>]\n";
+ print "Usage : agitator.pl <min sleep before kill in minutes>[:max sleep before kill in minutes] <min sleep before tup in minutes>[:max sleep before tup in minutes] [<min kill> <max kill>]\n";
exit(1);
}
$ACCUMULO_HOME="../../..";
$HADOOP_PREFIX=$ENV{"HADOOP_PREFIX"};
-$sleep1 = $ARGV[0];
-$sleep2 = $ARGV[1];
+@sleeprange1 = split(/:/, $ARGV[0]);
+$sleep1 = $sleeprange1[0];
+
+@sleeprange2 = split(/:/, $ARGV[1]);
+$sleep2 = $sleeprange2[0];
+
+if(scalar(@sleeprange1) > 1){
+ $sleep1max = $sleeprange1[1] + 1;
+}else{
+ $sleep1max = $sleep1;
+}
+
+if($sleep1 > $sleep1max){
+ die("sleep1 > sleep1max $sleep1 > $sleep1max");
+}
+
+if(scalar(@sleeprange2) > 1){
+ $sleep2max = $sleeprange2[1] + 1;
+}else{
+ $sleep2max = $sleep2;
+}
+
+if($sleep2 > $sleep2max){
+ die("sleep2 > sleep2max $sleep2 > $sleep2max");
+}
if(scalar(@ARGV) == 4){
$minKill = $ARGV[2];
@@ -104,13 +127,15 @@ while(1){
}
}
- sleep($sleep2 * 60);
+ $nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
+ sleep($nextsleep2 * 60);
$t = strftime "%Y%m%d %H:%M:%S", localtime;
print STDERR "$t Running tup\n";
system("$ACCUMULO_HOME/bin/tup.sh");
print STDERR "$t Running start-dfs\n";
system("$HADOOP_PREFIX/bin/start-dfs.sh");
- sleep($sleep1 * 60);
+ $nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
+ sleep($nextsleep1 * 60);
}