You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/06/15 23:52:47 UTC
[1/5] accumulo git commit: ACCUMULO-3423 more fixes for replication
Repository: accumulo
Updated Branches:
refs/heads/master 67249ec2e -> b7a529b75
ACCUMULO-3423 more fixes for replication
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ecf2298e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ecf2298e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ecf2298e
Branch: refs/heads/master
Commit: ecf2298eb9b764c01edc3f16f0c0cb6e6c4006cb
Parents: 844166a
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 15 09:59:38 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 15 09:59:38 2015 -0400
----------------------------------------------------------------------
.../accumulo/server/log/WalStateManager.java | 8 ++++-
.../CloseWriteAheadLogReferences.java | 2 +-
.../java/org/apache/accumulo/master/Master.java | 11 +++++++
.../accumulo/master/TabletGroupWatcher.java | 4 ++-
.../test/replication/ReplicationIT.java | 31 +++++---------------
5 files changed, 29 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ecf2298e/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
index 1540938..52844c1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
@@ -34,6 +34,8 @@ import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/*
* This class governs the space in Zookeeper that advertises the status of Write-Ahead Logs
@@ -68,6 +70,8 @@ public class WalStateManager {
}
}
+ private static final Logger log = LoggerFactory.getLogger(WalStateManager.class);
+
public final static String ZWALS = "/wals";
public static enum WalState {
@@ -113,6 +117,7 @@ public class WalStateManager {
if (state == WalState.OPEN) {
policy = NodeExistsPolicy.FAIL;
}
+ log.debug("Setting {} to {}", path.getName(), state);
zoo.putPersistentData(root() + "/" + tsi.toString() + "/" + path.getName(), data, policy);
} catch (KeeperException | InterruptedException e) {
throw new WalMarkerException(e);
@@ -193,6 +198,7 @@ public class WalStateManager {
// garbage collector knows it's safe to remove the marker for a closed log
public void removeWalMarker(TServerInstance instance, UUID uuid) throws WalMarkerException {
try {
+ log.debug("Removing {}", uuid);
String path = root() + "/" + instance.toString() + "/" + uuid.toString();
zoo.delete(path, -1);
} catch (InterruptedException | KeeperException e) {
@@ -211,8 +217,8 @@ public class WalStateManager {
}
// tablet server can mark the log as closed (but still needed), for replication to begin
+ // master can mark a log as unreferenced after it has made log recovery markers on the tablets that need to be recovered
public void closeWal(TServerInstance instance, Path path) throws WalMarkerException {
updateState(instance, path, WalState.CLOSED);
}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ecf2298e/server/gc/src/main/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferences.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferences.java b/server/gc/src/main/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferences.java
index 8857939..0c09396 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferences.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferences.java
@@ -185,7 +185,7 @@ public class CloseWriteAheadLogReferences implements Runnable {
// We only want to clean up WALs (which is everything but rfiles) and only when
// metadata doesn't have a reference to the given WAL
- if (!status.getClosed() && !replFile.endsWith(RFILE_SUFFIX) && !isClosed) {
+ if (!status.getClosed() && !replFile.endsWith(RFILE_SUFFIX) && isClosed) {
try {
closeWal(bw, entry.getKey());
recordsClosed++;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ecf2298e/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 9a324fb..0cf84f2 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -97,6 +97,8 @@ import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.log.WalStateManager;
+import org.apache.accumulo.server.log.WalStateManager.WalMarkerException;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
@@ -1586,4 +1588,13 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
return new HashSet<TServerInstance>(serversToShutdown);
}
}
+
+ public void markDeadServerLogsAsClosed(Map<TServerInstance,List<Path>> logsForDeadServers) throws WalMarkerException {
+ WalStateManager mgr = new WalStateManager(this.inst, ZooReaderWriter.getInstance());
+ for (Entry<TServerInstance,List<Path>> server : logsForDeadServers.entrySet()) {
+ for (Path path : server.getValue()) {
+ mgr.closeWal(server.getKey(), path);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ecf2298e/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index d55781e..d2cbf62 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -69,6 +69,7 @@ import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.log.WalStateManager;
+import org.apache.accumulo.server.log.WalStateManager.WalMarkerException;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.state.Assignment;
import org.apache.accumulo.server.master.state.ClosableIterator;
@@ -748,12 +749,13 @@ class TabletGroupWatcher extends Daemon {
private void flushChanges(SortedMap<TServerInstance,TabletServerStatus> currentTServers, List<Assignment> assignments, List<Assignment> assigned,
List<TabletLocationState> assignedToDeadServers, Map<TServerInstance,List<Path>> logsForDeadServers, Map<KeyExtent,TServerInstance> unassigned)
- throws DistributedStoreException, TException {
+ throws DistributedStoreException, TException, WalMarkerException {
if (!assignedToDeadServers.isEmpty()) {
int maxServersToShow = min(assignedToDeadServers.size(), 100);
Master.log.debug(assignedToDeadServers.size() + " assigned to dead servers: " + assignedToDeadServers.subList(0, maxServersToShow) + "...");
Master.log.debug("logs for dead servers: " + logsForDeadServers);
store.unassign(assignedToDeadServers, logsForDeadServers);
+ this.master.markDeadServerLogsAsClosed(logsForDeadServers);
this.master.nextEvent.event("Marked %d tablets as unassigned because they don't have current servers", assignedToDeadServers.size());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ecf2298e/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
index 55379a4..e0a9121 100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@ -435,26 +435,17 @@ public class ReplicationIT extends ConfigurableMacBase {
Assert.assertTrue("Replication table did not exist", online);
Assert.assertTrue(ReplicationTable.isOnline(conn));
- conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.READ);
// Verify that we found a single replication record that's for table1
Scanner s = ReplicationTable.getScanner(conn);
StatusSection.limit(s);
- Iterator<Entry<Key,Value>> iter = s.iterator();
- attempts = 5;
- while (attempts > 0) {
- if (!iter.hasNext()) {
- s.close();
- Thread.sleep(1000);
- s = ReplicationTable.getScanner(conn);
- iter = s.iterator();
- attempts--;
- } else {
+ for (int i = 0; i < 5; i++) {
+ if (Iterators.size(s.iterator()) == 1) {
break;
}
+ Thread.sleep(1000);
}
- Assert.assertTrue(iter.hasNext());
- Entry<Key,Value> entry = iter.next();
+ Entry<Key,Value> entry = Iterators.getOnlyElement(s.iterator());
// We should at least find one status record for this table, we might find a second if another log was started from ingesting the data
Assert.assertEquals("Expected to find replication entry for " + table1, conn.tableOperations().tableIdMap().get(table1), entry.getKey()
.getColumnQualifier().toString());
@@ -469,23 +460,15 @@ public class ReplicationIT extends ConfigurableMacBase {
// After the commit on these mutations, we'll get a replication entry in accumulo.metadata for table2
// Don't want to compact table2 as it ultimately cause the entry in accumulo.metadata to be removed before we can verify it's there
- // After writing data, we'll get a replication table online
- Assert.assertTrue(ReplicationTable.isOnline(conn));
-
Set<String> tableIds = Sets.newHashSet(conn.tableOperations().tableIdMap().get(table1), conn.tableOperations().tableIdMap().get(table2));
Set<String> tableIdsForMetadata = Sets.newHashSet(tableIds);
+ List<Entry<Key,Value>> records = new ArrayList<>();
s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
s.setRange(MetadataSchema.ReplicationSection.getRange());
-
- List<Entry<Key,Value>> records = new ArrayList<>();
for (Entry<Key,Value> metadata : s) {
records.add(metadata);
- }
- s = ReplicationTable.getScanner(conn);
- StatusSection.limit(s);
- for (Entry<Key,Value> replication : s) {
- records.add(replication);
+ log.debug("Meta: {} => {}", metadata.getKey().toStringNoTruncate(), metadata.getValue().toString());
}
Assert.assertEquals("Expected to find 2 records, but actually found " + records, 2, records.size());
@@ -503,7 +486,7 @@ public class ReplicationIT extends ConfigurableMacBase {
// Verify that we found two replication records: one for table1 and one for table2
s = ReplicationTable.getScanner(conn);
StatusSection.limit(s);
- iter = s.iterator();
+ Iterator<Entry<Key,Value>> iter = s.iterator();
Assert.assertTrue("Found no records in replication table", iter.hasNext());
entry = iter.next();
Assert.assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
[5/5] accumulo git commit: ACCUMULO-3871 fixed up a few more tests
Posted by ec...@apache.org.
ACCUMULO-3871 fixed up a few more tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b7a529b7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b7a529b7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b7a529b7
Branch: refs/heads/master
Commit: b7a529b75e335c577c53c77dfd1aee38a1cadcf6
Parents: b5004a1
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 15 17:52:38 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 15 17:52:38 2015 -0400
----------------------------------------------------------------------
.../client/impl/ReplicationOperationsImpl.java | 1 -
mrit/.gitignore | 1 +
.../accumulo/test/AccumuloOutputFormatIT.java | 73 +++++++-------------
.../accumulo/test/UserCompactionStrategyIT.java | 4 +-
.../apache/accumulo/test/functional/BulkIT.java | 12 +---
.../accumulo/test/functional/ClassLoaderIT.java | 3 +-
.../test/functional/GarbageCollectorIT.java | 19 +++--
.../apache/accumulo/test/functional/SslIT.java | 5 +-
.../test/performance/RollWALPerformanceIT.java | 9 +--
test/src/main/resources/log4j.properties | 2 +-
10 files changed, 50 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
index 925877d..009479e 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
@@ -175,7 +175,6 @@ public class ReplicationOperationsImpl implements ReplicationOperations {
} finally {
metaBs.close();
}
-
return wals;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/mrit/.gitignore
----------------------------------------------------------------------
diff --git a/mrit/.gitignore b/mrit/.gitignore
index 56204d2..db98860 100644
--- a/mrit/.gitignore
+++ b/mrit/.gitignore
@@ -15,6 +15,7 @@
# Maven ignores
/target/
+/dependency-reduced-pom.xml
# IDE ignores
/.settings/
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java b/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java
index a2f522e..6b5e6eb 100644
--- a/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/AccumuloOutputFormatIT.java
@@ -16,75 +16,46 @@
*/
package org.apache.accumulo.test;
-import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
-import java.io.File;
import java.io.IOException;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.mapred.AccumuloOutputFormat;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.minicluster.MiniAccumuloCluster;
-import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordWriter;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-
-import com.google.common.collect.Maps;
/**
- * Prevent regression of ACCUMULO-3709. Exists as a mini test because mock instance doesn't produce this error when dynamically changing the table permissions.
+ * Prevent regression of ACCUMULO-3709.
*/
-public class AccumuloOutputFormatIT {
+public class AccumuloOutputFormatIT extends ConfigurableMacBase {
private static final String TABLE = "abc";
- private MiniAccumuloCluster accumulo;
- private String secret = "secret";
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- @Before
- public void setUp() throws Exception {
- folder.create();
- MiniAccumuloConfig config = new MiniAccumuloConfig(folder.getRoot(), secret);
- Map<String,String> configMap = Maps.newHashMap();
- configMap.put(Property.TSERV_SESSION_MAXIDLE.toString(), "1");
- config.setSiteConfig(configMap);
- config.setNumTservers(1);
- accumulo = new MiniAccumuloCluster(config);
- accumulo.start();
- }
-
- @After
- public void tearDown() throws Exception {
- accumulo.stop();
- folder.delete();
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setProperty(Property.TSERV_SESSION_MAXIDLE, "1");
+ cfg.setNumTservers(1);
}
@Test
public void testMapred() throws Exception {
- ClientConfiguration clientConfig = accumulo.getClientConfig();
- ZooKeeperInstance instance = new ZooKeeperInstance(clientConfig);
- Connector connector = instance.getConnector("root", new PasswordToken(secret));
+ Connector connector = getConnector();
// create a table and put some data in it
connector.tableOperations().create(TABLE);
@@ -98,8 +69,8 @@ public class AccumuloOutputFormatIT {
batchConfig.setMaxMemory(Long.MAX_VALUE);
AccumuloOutputFormat outputFormat = new AccumuloOutputFormat();
AccumuloOutputFormat.setBatchWriterOptions(job, batchConfig);
- AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig);
- AccumuloOutputFormat.setConnectorInfo(job, "root", new PasswordToken(secret));
+ AccumuloOutputFormat.setZooKeeperInstance(job, cluster.getClientConfig());
+ AccumuloOutputFormat.setConnectorInfo(job, "root", new PasswordToken(ROOT_PASSWORD));
RecordWriter<Text,Mutation> writer = outputFormat.getRecordWriter(null, job, "Test", null);
try {
@@ -107,8 +78,8 @@ public class AccumuloOutputFormatIT {
Mutation m = new Mutation(new Text(String.format("%08d", i)));
for (int j = 0; j < 3; j++) {
m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
- writer.write(new Text(TABLE), m);
}
+ writer.write(new Text(TABLE), m);
}
} catch (Exception e) {
@@ -118,8 +89,12 @@ public class AccumuloOutputFormatIT {
connector.securityOperations().revokeTablePermission("root", TABLE, TablePermission.WRITE);
- exception.expect(IOException.class);
- exception.expectMessage("PERMISSION_DENIED");
- writer.close(null);
+ try {
+ writer.close(null);
+ fail("Did not throw exception");
+ } catch (IOException ex) {
+ log.info(ex.getMessage(), ex);
+ assertTrue(ex.getCause() instanceof MutationsRejectedException);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java b/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java
index 9c319d2..588a83b 100644
--- a/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/UserCompactionStrategyIT.java
@@ -128,12 +128,12 @@ public class UserCompactionStrategyIT extends AccumuloClusterHarness {
// Can't assume that a test-resource will be on the server's classpath
Assume.assumeTrue(ClusterType.MINI == getClusterType());
- // test pertable classpath + user specified compaction strat
+ // test per-table classpath + user specified compaction strategy
final Connector c = getConnector();
final String tableName = getUniqueNames(1)[0];
File target = new File(System.getProperty("user.dir"), "target");
- Assert.assertTrue(target.mkdirs());
+ target.mkdirs();
Assert.assertTrue(target.exists() && target.isDirectory());
File destFile = installJar(target, "/TestCompactionStrat.jar");
c.tableOperations().create(tableName);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java
index f60724e..04570a4 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkIT.java
@@ -26,10 +26,8 @@ import org.apache.accumulo.test.TestIngest.Opts;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -67,7 +65,6 @@ public class BulkIT extends AccumuloClusterHarness {
static void runTest(Connector c, FileSystem fs, Path basePath, String principal, String tableName, String filePrefix, String dirSuffix) throws Exception {
c.tableOperations().create(tableName);
- CachedConfiguration.setInstance(fs.getConf());
Path base = new Path(basePath, "testBulkFail_" + dirSuffix);
fs.delete(base, true);
@@ -84,24 +81,21 @@ public class BulkIT extends AccumuloClusterHarness {
opts.instance = c.getInstance().getInstanceName();
opts.cols = 1;
opts.setTableName(tableName);
- opts.conf = CachedConfiguration.getInstance();
+ opts.conf = new Configuration(false);
opts.fs = fs;
String fileFormat = filePrefix + "rf%02d";
for (int i = 0; i < COUNT; i++) {
opts.outputFile = new Path(files, String.format(fileFormat, i)).toString();
opts.startRow = N * i;
- TestIngest.ingest(c, opts, BWOPTS);
+ TestIngest.ingest(c, fs, opts, BWOPTS);
}
opts.outputFile = base + String.format(fileFormat, N);
opts.startRow = N;
opts.rows = 1;
// create an rfile with one entry, there was a bug with this:
- TestIngest.ingest(c, opts, BWOPTS);
+ TestIngest.ingest(c, fs, opts, BWOPTS);
// Make sure the server can modify the files
- FsShell fsShell = new FsShell(fs.getConf());
- Assert.assertEquals("Failed to chmod " + base.toString(), 0, fsShell.run(new String[] {"-chmod", "-R", "777", base.toString()}));
-
c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), false);
VerifyIngest.Opts vopts = new VerifyIngest.Opts();
vopts.setTableName(tableName);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java
index f1fb91c..ebb0950 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ClassLoaderIT.java
@@ -38,7 +38,6 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Combiner;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
@@ -92,7 +91,7 @@ public class ClassLoaderIT extends AccumuloClusterHarness {
bw.addMutation(m);
bw.close();
scanCheck(c, tableName, "Test");
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ FileSystem fs = getCluster().getFileSystem();
Path jarPath = new Path(rootPath + "/lib/ext/Test.jar");
copyStreamToFileSystem(fs, this.getClass().getResourceAsStream("/TestCombinerX.jar"), jarPath);
UtilWaitThread.sleep(1000);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index a73f239..6de8e93 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.test.functional;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -43,7 +44,6 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ServerServices.Service;
import org.apache.accumulo.core.util.UtilWaitThread;
@@ -59,7 +59,6 @@ import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
@@ -119,7 +118,7 @@ public class GarbageCollectorIT extends ConfigurableMacBase {
vopts.cols = opts.cols = 1;
opts.setPrincipal("root");
vopts.setPrincipal("root");
- TestIngest.ingest(c, opts, new BatchWriterOpts());
+ TestIngest.ingest(c, cluster.getFileSystem(), opts, new BatchWriterOpts());
c.tableOperations().compact("test_ingest", null, null, true, true);
int before = countFiles();
while (true) {
@@ -148,7 +147,16 @@ public class GarbageCollectorIT extends ConfigurableMacBase {
cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE);
Process gc = cluster.exec(SimpleGarbageCollector.class);
UtilWaitThread.sleep(20 * 1000);
- String output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc);
+ String output = "";
+ while (!output.contains("delete candidates has exceeded")) {
+ byte buffer[] = new byte[10 * 1024];
+ try {
+ int n = gc.getInputStream().read(buffer);
+ output = new String(buffer, 0, n, UTF_8);
+ } catch (IOException ex) {
+ break;
+ }
+ }
gc.destroy();
assertTrue(output.contains("delete candidates has exceeded"));
}
@@ -279,9 +287,8 @@ public class GarbageCollectorIT extends ConfigurableMacBase {
}
private int countFiles() throws Exception {
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
Path path = new Path(cluster.getConfig().getDir() + "/accumulo/tables/1/*/*.rf");
- return Iterators.size(Arrays.asList(fs.globStatus(path)).iterator());
+ return Iterators.size(Arrays.asList(cluster.getFileSystem().globStatus(path)).iterator());
}
public static void addEntries(Connector conn, BatchWriterOpts bwOpts) throws Exception {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java
index 13248d0..b81b409 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SslIT.java
@@ -20,7 +20,6 @@ import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
@@ -60,8 +59,8 @@ public class SslIT extends ConfigurableMacBase {
@Test
public void bulk() throws Exception {
- BulkIT.runTest(getConnector(), FileSystem.getLocal(new Configuration(false)), new Path(getCluster().getConfig().getDir().getAbsolutePath(), "tmp"), "root",
- getUniqueNames(1)[0], this.getClass().getName(), testName.getMethodName());
+ BulkIT.runTest(getConnector(), cluster.getFileSystem(), new Path(getCluster().getConfig().getDir().getAbsolutePath(), "tmp"), "root", getUniqueNames(1)[0],
+ this.getClass().getName(), testName.getMethodName());
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
index a0d355e..bd42989 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
@@ -28,7 +28,6 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
import org.apache.accumulo.test.continuous.ContinuousIngest;
import org.apache.accumulo.test.functional.ConfigurableMacBase;
import org.apache.hadoop.conf.Configuration;
@@ -40,7 +39,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase {
@Override
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1");
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "10M");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100");
cfg.setProperty(Property.GC_FILE_ARCHIVE, "false");
cfg.setProperty(Property.GC_CYCLE_START, "1s");
@@ -72,7 +71,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase {
log.info("Starting ingest");
final long start = System.currentTimeMillis();
final String args[] = {"-i", inst.getInstanceName(), "-z", inst.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--batchThreads", "2", "--table",
- tableName, "--num", Long.toString(1000 * 1000), // 1M 100 byte entries
+ tableName, "--num", Long.toString(50 * 1000), // 50K 100 byte entries
};
ContinuousIngest.main(args);
@@ -100,9 +99,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase {
c.instanceOperations().setProperty(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1G");
c.tableOperations().flush(MetadataTable.NAME, null, null, true);
c.tableOperations().flush(RootTable.NAME, null, null, true);
- for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
- getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
- }
+ getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
getCluster().start();
long avg2 = getAverage();
log.info(String.format("Average run time with small WAL %,d with large WAL %,d", avg1, avg2));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7a529b7/test/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/test/src/main/resources/log4j.properties b/test/src/main/resources/log4j.properties
index 26ea762..c603b39 100644
--- a/test/src/main/resources/log4j.properties
+++ b/test/src/main/resources/log4j.properties
@@ -43,7 +43,7 @@ log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
log4j.logger.org.apache.hadoop.hdfs.server.datanode.DataNode.clienttrace=WARN
log4j.logger.BlockStateChange=WARN
log4j.logger.org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator=INFO
-log4j.logger.org.apache.hadoop.security=DEBUG
+log4j.logger.org.apache.hadoop.security=INFO
log4j.logger.org.apache.hadoop.minikdc=DEBUG
log4j.logger.org.apache.directory=INFO
log4j.logger.org.apache.directory.api.ldap=WARN
[4/5] accumulo git commit: ACCUMULO-3423 mark logs as closed when
tablet servers die
Posted by ec...@apache.org.
ACCUMULO-3423 mark logs as closed when tablet servers die
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b5004a1c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b5004a1c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b5004a1c
Branch: refs/heads/master
Commit: b5004a1c4c518f89deb457e987f228ccf9bf0d5b
Parents: e446f0a
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 15 17:52:24 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 15 17:52:24 2015 -0400
----------------------------------------------------------------------
server/master/src/main/java/org/apache/accumulo/master/Master.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b5004a1c/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 0cf84f2..95f3f4e 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -1593,7 +1593,7 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
WalStateManager mgr = new WalStateManager(this.inst, ZooReaderWriter.getInstance());
for (Entry<TServerInstance,List<Path>> server : logsForDeadServers.entrySet()) {
for (Path path : server.getValue()) {
- mgr.closeWal(server.getKey(), path);
+ mgr.closeWal(server.getKey(), path);
}
}
}
[3/5] accumulo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/accumulo
Posted by ec...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e446f0ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e446f0ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e446f0ad
Branch: refs/heads/master
Commit: e446f0ad960ac41fc5c6bef9f865d22419fd958e
Parents: 9e89b65 67249ec
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 15 17:50:57 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 15 17:50:57 2015 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/tracer/TraceServer.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[2/5] accumulo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/accumulo
Posted by ec...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9e89b65d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9e89b65d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9e89b65d
Branch: refs/heads/master
Commit: 9e89b65d5068aef706894b211bff4c62ed16259c
Parents: ecf2298 2e51eec
Author: Eric C. Newton <er...@gmail.com>
Authored: Mon Jun 15 09:59:49 2015 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Mon Jun 15 09:59:49 2015 -0400
----------------------------------------------------------------------
.../core/client/lexicoder/AbstractEncoder.java | 2 +-
.../lexicoder/impl/AbstractLexicoderTest.java | 5 ----
mrit/.gitignore | 1 -
mrit/pom.xml | 2 --
pom.xml | 5 ++++
.../accumulo/test/proxy/SimpleProxyBase.java | 29 +++++++++++---------
6 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------