You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/06/04 20:53:14 UTC
[33/43] accumulo git commit: ACCUMULO-3871 move ITs into distro jar,
stop building test jar
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java b/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java
new file mode 100644
index 0000000..2251d4b
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.fail;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.trace.Tracer;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DynamicThreadPoolsIT extends AccumuloClusterHarness {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ Map<String,String> siteConfig = cfg.getSiteConfig();
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private String majcDelay;
+
+ @Before
+ public void updateMajcDelay() throws Exception {
+ Connector c = getConnector();
+ majcDelay = c.instanceOperations().getSystemConfiguration().get(Property.TSERV_MAJC_DELAY.getKey());
+ c.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
+ if (getClusterType() == ClusterType.STANDALONE) {
+ Thread.sleep(AccumuloConfiguration.getTimeInMillis(majcDelay));
+ }
+ }
+
+ @After
+ public void resetMajcDelay() throws Exception {
+ Connector c = getConnector();
+ c.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), majcDelay);
+ }
+
+ @Test
+ public void test() throws Exception {
+ final String[] tables = getUniqueNames(15);
+ String firstTable = tables[0];
+ Connector c = getConnector();
+ c.instanceOperations().setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "5");
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.rows = 500 * 1000;
+ opts.createTable = true;
+ opts.setTableName(firstTable);
+ ClientConfiguration clientConf = cluster.getClientConfig();
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConf);
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ c.tableOperations().flush(firstTable, null, null, true);
+ for (int i = 1; i < tables.length; i++)
+ c.tableOperations().clone(firstTable, tables[i], true, null, null);
+ UtilWaitThread.sleep(11 * 1000); // time between checks of the thread pool sizes
+ Credentials creds = new Credentials(getAdminPrincipal(), getAdminToken());
+ for (int i = 1; i < tables.length; i++)
+ c.tableOperations().compact(tables[i], null, null, true, false);
+ for (int i = 0; i < 30; i++) {
+ int count = 0;
+ MasterClientService.Iface client = null;
+ MasterMonitorInfo stats = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(new ClientContext(c.getInstance(), creds, clientConf));
+ stats = client.getMasterStats(Tracer.traceInfo(), creds.toThrift(c.getInstance()));
+ } finally {
+ if (client != null)
+ MasterClient.close(client);
+ }
+ for (TabletServerStatus server : stats.tServerInfo) {
+ for (TableInfo table : server.tableMap.values()) {
+ count += table.majors.running;
+ }
+ }
+ System.out.println("count " + count);
+ if (count > 3)
+ return;
+ UtilWaitThread.sleep(500);
+ }
+ fail("Could not observe higher number of threads after changing the config");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
new file mode 100644
index 0000000..9d0ce86
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@ -0,0 +1,660 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster;
+import org.apache.accumulo.cluster.standalone.StandaloneClusterControl;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.client.Flush;
+import org.apache.accumulo.examples.simple.client.RandomBatchScanner;
+import org.apache.accumulo.examples.simple.client.RandomBatchWriter;
+import org.apache.accumulo.examples.simple.client.ReadWriteExample;
+import org.apache.accumulo.examples.simple.client.RowOperations;
+import org.apache.accumulo.examples.simple.client.SequentialBatchWriter;
+import org.apache.accumulo.examples.simple.client.TraceDumpExample;
+import org.apache.accumulo.examples.simple.client.TracingExample;
+import org.apache.accumulo.examples.simple.combiner.StatsCombiner;
+import org.apache.accumulo.examples.simple.constraints.MaxMutationSize;
+import org.apache.accumulo.examples.simple.dirlist.Ingest;
+import org.apache.accumulo.examples.simple.dirlist.QueryUtil;
+import org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter;
+import org.apache.accumulo.examples.simple.helloworld.ReadData;
+import org.apache.accumulo.examples.simple.isolation.InterferenceTest;
+import org.apache.accumulo.examples.simple.mapreduce.RegexExample;
+import org.apache.accumulo.examples.simple.mapreduce.RowHash;
+import org.apache.accumulo.examples.simple.mapreduce.TableToFile;
+import org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest;
+import org.apache.accumulo.examples.simple.mapreduce.WordCount;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest;
+import org.apache.accumulo.examples.simple.shard.ContinuousQuery;
+import org.apache.accumulo.examples.simple.shard.Index;
+import org.apache.accumulo.examples.simple.shard.Query;
+import org.apache.accumulo.examples.simple.shard.Reverse;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.LogWriter;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.start.Main;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.Tool;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class ExamplesIT extends AccumuloClusterHarness {
+ private static final Logger log = LoggerFactory.getLogger(ExamplesIT.class);
+ private static final BatchWriterOpts bwOpts = new BatchWriterOpts();
+ private static final BatchWriterConfig bwc = new BatchWriterConfig();
+ private static final String visibility = "A|B";
+ private static final String auths = "A,B";
+
+ Connector c;
+ String instance;
+ String keepers;
+ String user;
+ String passwd;
+ String keytab;
+ BatchWriter bw;
+ IteratorSetting is;
+ String dir;
+ FileSystem fs;
+ Authorizations origAuths;
+ boolean saslEnabled;
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopConf) {
+ // 128MB * 3
+ cfg.setDefaultMemory(cfg.getDefaultMemory() * 3, MemoryUnit.BYTE);
+ }
+
+ @Before
+ public void getClusterInfo() throws Exception {
+ c = getConnector();
+ user = getAdminPrincipal();
+ AuthenticationToken token = getAdminToken();
+ if (token instanceof KerberosToken) {
+ keytab = getAdminUser().getKeytab().getAbsolutePath();
+ saslEnabled = true;
+ } else if (token instanceof PasswordToken) {
+ passwd = new String(((PasswordToken) getAdminToken()).getPassword(), UTF_8);
+ saslEnabled = false;
+ } else {
+ Assert.fail("Unknown token type: " + token);
+ }
+ fs = getCluster().getFileSystem();
+ instance = c.getInstance().getInstanceName();
+ keepers = c.getInstance().getZooKeepers();
+ dir = new Path(cluster.getTemporaryPath(), getClass().getName()).toString();
+
+ origAuths = c.securityOperations().getUserAuthorizations(user);
+ c.securityOperations().changeUserAuthorizations(user, new Authorizations(auths.split(",")));
+ }
+
+ @After
+ public void resetAuths() throws Exception {
+ if (null != origAuths) {
+ getConnector().securityOperations().changeUserAuthorizations(getAdminPrincipal(), origAuths);
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 6 * 60;
+ }
+
+ @Test
+ public void testTrace() throws Exception {
+ Process trace = null;
+ if (ClusterType.MINI == getClusterType()) {
+ MiniAccumuloClusterImpl impl = (MiniAccumuloClusterImpl) cluster;
+ trace = impl.exec(TraceServer.class);
+ while (!c.tableOperations().exists("trace"))
+ UtilWaitThread.sleep(500);
+ }
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-C", "-D", "-c"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-C", "-D", "-c"};
+ }
+ Entry<Integer,String> pair = cluster.getClusterControl().execWithStdout(TracingExample.class, args);
+ Assert.assertEquals("Expected return code of zero. STDOUT=" + pair.getValue(), 0, pair.getKey().intValue());
+ String result = pair.getValue();
+ Pattern pattern = Pattern.compile("TraceID: ([0-9a-f]+)");
+ Matcher matcher = pattern.matcher(result);
+ int count = 0;
+ while (matcher.find()) {
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--traceid", matcher.group(1)};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--traceid", matcher.group(1)};
+ }
+ pair = cluster.getClusterControl().execWithStdout(TraceDumpExample.class, args);
+ count++;
+ }
+ assertTrue(count > 0);
+ assertTrue("Output did not contain myApp@myHost", pair.getValue().contains("myApp@myHost"));
+ if (ClusterType.MINI == getClusterType() && null != trace) {
+ trace.destroy();
+ }
+ }
+
+ @Test
+ public void testClasspath() throws Exception {
+ Entry<Integer,String> entry = getCluster().getClusterControl().execWithStdout(Main.class, new String[] {"classpath"});
+ assertEquals(0, entry.getKey().intValue());
+ String result = entry.getValue();
+ int level1 = result.indexOf("Level 1");
+ int level2 = result.indexOf("Level 2");
+ int level3 = result.indexOf("Level 3");
+ int level4 = result.indexOf("Level 4");
+ assertTrue("Level 1 classloader not present.", level1 >= 0);
+ assertTrue("Level 2 classloader not present.", level2 > 0);
+ assertTrue("Level 3 classloader not present.", level3 > 0);
+ assertTrue("Level 4 classloader not present.", level4 > 0);
+ assertTrue(level1 < level2);
+ assertTrue(level2 < level3);
+ assertTrue(level3 < level4);
+ }
+
+ @Test
+ public void testDirList() throws Exception {
+ String[] names = getUniqueNames(3);
+ String dirTable = names[0], indexTable = names[1], dataTable = names[2];
+ String[] args;
+ String dirListDirectory;
+ switch (getClusterType()) {
+ case MINI:
+ dirListDirectory = ((MiniAccumuloClusterImpl) getCluster()).getConfig().getDir().getAbsolutePath();
+ break;
+ case STANDALONE:
+ dirListDirectory = ((StandaloneAccumuloCluster) getCluster()).getAccumuloHome();
+ break;
+ default:
+ throw new RuntimeException("Unknown cluster type");
+ }
+ // Index a directory listing on /tmp. If this is running against a standalone cluster, we can't guarantee Accumulo source will be there.
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--dirTable", dirTable, "--indexTable", indexTable, "--dataTable",
+ dataTable, "--vis", visibility, "--chunkSize", Integer.toString(10000), dirListDirectory};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--dirTable", dirTable, "--indexTable", indexTable, "--dataTable",
+ dataTable, "--vis", visibility, "--chunkSize", Integer.toString(10000), dirListDirectory};
+ }
+ Entry<Integer,String> entry = getClusterControl().execWithStdout(Ingest.class, args);
+ assertEquals("Got non-zero return code. Stdout=" + entry.getValue(), 0, entry.getKey().intValue());
+
+ String expectedFile;
+ switch (getClusterType()) {
+ case MINI:
+ // Should be present in a minicluster dir
+ expectedFile = "accumulo-site.xml";
+ break;
+ case STANDALONE:
+ // Should be in place on standalone installs (not having ot follow symlinks)
+ expectedFile = "LICENSE";
+ break;
+ default:
+ throw new RuntimeException("Unknown cluster type");
+ }
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "--keytab", keytab, "-u", user, "-t", indexTable, "--auths", auths, "--search", "--path",
+ expectedFile};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-p", passwd, "-u", user, "-t", indexTable, "--auths", auths, "--search", "--path", expectedFile};
+ }
+ entry = getClusterControl().execWithStdout(QueryUtil.class, args);
+ if (ClusterType.MINI == getClusterType()) {
+ MiniAccumuloClusterImpl impl = (MiniAccumuloClusterImpl) cluster;
+ for (LogWriter writer : impl.getLogWriters()) {
+ writer.flush();
+ }
+ }
+
+ log.info("result " + entry.getValue());
+ assertEquals(0, entry.getKey().intValue());
+ assertTrue(entry.getValue().contains(expectedFile));
+ }
+
+ @Test
+ public void testAgeoffFilter() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ is = new IteratorSetting(10, AgeOffFilter.class);
+ AgeOffFilter.setTTL(is, 1000L);
+ c.tableOperations().attachIterator(tableName, is);
+ UtilWaitThread.sleep(500); // let zookeeper updates propagate.
+ bw = c.createBatchWriter(tableName, bwc);
+ Mutation m = new Mutation("foo");
+ m.put("a", "b", "c");
+ bw.addMutation(m);
+ bw.close();
+ UtilWaitThread.sleep(1000);
+ assertEquals(0, Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator()));
+ }
+
+ @Test
+ public void testStatsCombiner() throws Exception {
+ String table = getUniqueNames(1)[0];
+ c.tableOperations().create(table);
+ is = new IteratorSetting(10, StatsCombiner.class);
+ StatsCombiner.setCombineAllColumns(is, true);
+
+ c.tableOperations().attachIterator(table, is);
+ bw = c.createBatchWriter(table, bwc);
+ // Write two mutations otherwise the NativeMap would dedupe them into a single update
+ Mutation m = new Mutation("foo");
+ m.put("a", "b", "1");
+ bw.addMutation(m);
+ m = new Mutation("foo");
+ m.put("a", "b", "3");
+ bw.addMutation(m);
+ bw.flush();
+
+ Iterator<Entry<Key,Value>> iter = c.createScanner(table, Authorizations.EMPTY).iterator();
+ assertTrue("Iterator had no results", iter.hasNext());
+ Entry<Key,Value> e = iter.next();
+ assertEquals("Results ", "1,3,4,2", e.getValue().toString());
+ assertFalse("Iterator had additional results", iter.hasNext());
+
+ m = new Mutation("foo");
+ m.put("a", "b", "0,20,20,2");
+ bw.addMutation(m);
+ bw.close();
+
+ iter = c.createScanner(table, Authorizations.EMPTY).iterator();
+ assertTrue("Iterator had no results", iter.hasNext());
+ e = iter.next();
+ assertEquals("Results ", "0,20,24,4", e.getValue().toString());
+ assertFalse("Iterator had additional results", iter.hasNext());
+ }
+
+ @Test
+ public void testBloomFilters() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"--seed", "7", "-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--num", "100000", "--min", "0", "--max",
+ "1000000000", "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", tableName};
+ } else {
+ args = new String[] {"--seed", "7", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "100000", "--min", "0", "--max", "1000000000",
+ "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", tableName};
+ }
+ goodExec(RandomBatchWriter.class, args);
+ c.tableOperations().flush(tableName, null, null, true);
+ long diff = 0, diff2 = 0;
+ // try the speed test a couple times in case the system is loaded with other tests
+ for (int i = 0; i < 2; i++) {
+ long now = System.currentTimeMillis();
+ if (saslEnabled) {
+ args = new String[] {"--seed", "7", "-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--num", "10000", "--min", "0", "--max",
+ "1000000000", "--size", "50", "--scanThreads", "4", "-t", tableName};
+ } else {
+ args = new String[] {"--seed", "7", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "10000", "--min", "0", "--max", "1000000000",
+ "--size", "50", "--scanThreads", "4", "-t", tableName};
+ }
+ goodExec(RandomBatchScanner.class, args);
+ diff = System.currentTimeMillis() - now;
+ now = System.currentTimeMillis();
+ if (saslEnabled) {
+ args = new String[] {"--seed", "8", "-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--num", "10000", "--min", "0", "--max",
+ "1000000000", "--size", "50", "--scanThreads", "4", "-t", tableName};
+ } else {
+ args = new String[] {"--seed", "8", "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--num", "10000", "--min", "0", "--max", "1000000000",
+ "--size", "50", "--scanThreads", "4", "-t", tableName};
+ }
+ int retCode = getClusterControl().exec(RandomBatchScanner.class, args);
+ assertEquals(1, retCode);
+ diff2 = System.currentTimeMillis() - now;
+ if (diff2 < diff)
+ break;
+ }
+ assertTrue(diff2 < diff);
+ }
+
+ @Test
+ public void testShardedIndex() throws Exception {
+ String[] names = getUniqueNames(3);
+ final String shard = names[0], index = names[1];
+ c.tableOperations().create(shard);
+ c.tableOperations().create(index);
+ bw = c.createBatchWriter(shard, bwc);
+ Index.index(30, new File(System.getProperty("user.dir") + "/src"), "\\W+", bw);
+ bw.close();
+ BatchScanner bs = c.createBatchScanner(shard, Authorizations.EMPTY, 4);
+ List<String> found = Query.query(bs, Arrays.asList("foo", "bar"));
+ bs.close();
+ // should find ourselves
+ boolean thisFile = false;
+ for (String file : found) {
+ if (file.endsWith("/ExamplesIT.java"))
+ thisFile = true;
+ }
+ assertTrue(thisFile);
+
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "--shardTable", shard, "--doc2Term", index, "-u", user, "--keytab", keytab};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "--shardTable", shard, "--doc2Term", index, "-u", getAdminPrincipal(), "-p", passwd};
+ }
+ // create a reverse index
+ goodExec(Reverse.class, args);
+
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "--shardTable", shard, "--doc2Term", index, "-u", user, "--keytab", keytab, "--terms", "5",
+ "--count", "1000"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "--shardTable", shard, "--doc2Term", index, "-u", user, "-p", passwd, "--terms", "5", "--count",
+ "1000"};
+ }
+ // run some queries
+ goodExec(ContinuousQuery.class, args);
+ }
+
+ @Test
+ public void testMaxMutationConstraint() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().addConstraint(tableName, MaxMutationSize.class.getName());
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.rows = 1;
+ opts.cols = 1000;
+ opts.setTableName(tableName);
+ if (saslEnabled) {
+ opts.updateKerberosCredentials(cluster.getClientConfig());
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ }
+ try {
+ TestIngest.ingest(c, opts, bwOpts);
+ } catch (MutationsRejectedException ex) {
+ assertEquals(1, ex.getConstraintViolationSummaries().size());
+ }
+ }
+
+ @Test
+ public void testBulkIngest() throws Exception {
+ // TODO Figure out a way to run M/R with Kerberos
+ Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+ String tableName = getUniqueNames(1)[0];
+ FileSystem fs = getFileSystem();
+ Path p = new Path(dir, "tmp");
+ if (fs.exists(p)) {
+ fs.delete(p, true);
+ }
+ goodExec(GenerateTestData.class, "--start-row", "0", "--count", "10000", "--output", dir + "/tmp/input/data");
+
+ List<String> commonArgs = new ArrayList<>(Arrays.asList(new String[] {"-i", instance, "-z", keepers, "-u", user, "--table", tableName}));
+ if (saslEnabled) {
+ commonArgs.add("--keytab");
+ commonArgs.add(keytab);
+ } else {
+ commonArgs.add("-p");
+ commonArgs.add(passwd);
+ }
+
+ List<String> args = new ArrayList<>(commonArgs);
+ goodExec(SetupTable.class, args.toArray(new String[0]));
+
+ args = new ArrayList<>(commonArgs);
+ args.addAll(Arrays.asList(new String[] {"--inputDir", dir + "/tmp/input", "--workDir", dir + "/tmp"}));
+ goodExec(BulkIngestExample.class, args.toArray(new String[0]));
+
+ args = new ArrayList<>(commonArgs);
+ args.addAll(Arrays.asList(new String[] {"--start-row", "0", "--count", "10000"}));
+ goodExec(VerifyIngest.class, args.toArray(new String[0]));
+ }
+
+ @Test
+ public void testTeraSortAndRead() throws Exception {
+ // TODO Figure out a way to run M/R with Kerberos
+ Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+ String tableName = getUniqueNames(1)[0];
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10", "-nv", "10", "-xv", "10", "-t", tableName, "-i", instance, "-z", keepers,
+ "-u", user, "--keytab", keytab, "--splits", "4"};
+ } else {
+ args = new String[] {"--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10", "-nv", "10", "-xv", "10", "-t", tableName, "-i", instance, "-z", keepers,
+ "-u", user, "-p", passwd, "--splits", "4"};
+ }
+ goodExec(TeraSortIngest.class, args);
+ Path output = new Path(dir, "tmp/nines");
+ if (fs.exists(output)) {
+ fs.delete(output, true);
+ }
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", tableName, "--rowRegex", ".*999.*", "--output",
+ output.toString()};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--rowRegex", ".*999.*", "--output", output.toString()};
+ }
+ goodExec(RegexExample.class, args);
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", tableName, "--column", "c:"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--column", "c:"};
+ }
+ goodExec(RowHash.class, args);
+ output = new Path(dir, "tmp/tableFile");
+ if (fs.exists(output)) {
+ fs.delete(output, true);
+ }
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", tableName, "--output", output.toString()};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--output", output.toString()};
+ }
+ goodExec(TableToFile.class, args);
+ }
+
+ @Test
+ public void testWordCount() throws Exception {
+ // TODO Figure out a way to run M/R with Kerberos
+ Assume.assumeTrue(getAdminToken() instanceof PasswordToken);
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new Text("count"))));
+ SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
+ c.tableOperations().attachIterator(tableName, is);
+ fs.copyFromLocalFile(new Path(new Path(System.getProperty("user.dir")).getParent(), "README.md"), new Path(dir + "/tmp/wc/README.md"));
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-u", user, "--keytab", keytab, "-z", keepers, "--input", dir + "/tmp/wc", "-t", tableName};
+ } else {
+ args = new String[] {"-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input", dir + "/tmp/wc", "-t", tableName};
+ }
+ goodExec(WordCount.class, args);
+ }
+
+ @Test
+ public void testInsertWithBatchWriterAndReadData() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", tableName};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName};
+ }
+ goodExec(InsertWithBatchWriter.class, args);
+ goodExec(ReadData.class, args);
+ }
+
+ @Test
+ public void testIsolatedScansWithInterference() throws Exception {
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", getUniqueNames(1)[0], "--iterations", "100000", "--isolated"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", getUniqueNames(1)[0], "--iterations", "100000", "--isolated"};
+ }
+ goodExec(InterferenceTest.class, args);
+ }
+
+ @Test
+ public void testScansWithInterference() throws Exception {
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", getUniqueNames(1)[0], "--iterations", "100000"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", getUniqueNames(1)[0], "--iterations", "100000"};
+ }
+ goodExec(InterferenceTest.class, args);
+ }
+
+ @Test
+ public void testRowOperations() throws Exception {
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd};
+ }
+ goodExec(RowOperations.class, args);
+ }
+
+ @Test
+ public void testBatchWriter() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "-t", tableName, "--start", "0", "--num", "100000", "--size", "50",
+ "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", tableName, "--start", "0", "--num", "100000", "--size", "50",
+ "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility};
+ }
+ goodExec(SequentialBatchWriter.class, args);
+
+ }
+
+ @Test
+ public void testReadWriteAndDelete() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--auths", auths, "--table", tableName, "--createtable", "-c",
+ "--debug"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", tableName, "--createtable", "-c", "--debug"};
+ }
+ goodExec(ReadWriteExample.class, args);
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--auths", auths, "--table", tableName, "-d", "--debug"};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", tableName, "-d", "--debug"};
+ }
+ goodExec(ReadWriteExample.class, args);
+
+ }
+
+ @Test
+ public void testRandomBatchesAndFlush() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ String[] args;
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--table", tableName, "--num", "100000", "--min", "0", "--max",
+ "100000", "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName, "--num", "100000", "--min", "0", "--max", "100000",
+ "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility};
+ }
+ goodExec(RandomBatchWriter.class, args);
+
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--table", tableName, "--num", "10000", "--min", "0", "--max",
+ "100000", "--size", "100", "--scanThreads", "4", "--auths", auths};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName, "--num", "10000", "--min", "0", "--max", "100000",
+ "--size", "100", "--scanThreads", "4", "--auths", auths};
+ }
+ goodExec(RandomBatchScanner.class, args);
+
+ if (saslEnabled) {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "--keytab", keytab, "--table", tableName};
+ } else {
+ args = new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", tableName};
+ }
+ goodExec(Flush.class, args);
+ }
+
+ private void goodExec(Class<?> theClass, String... args) throws InterruptedException, IOException {
+ Entry<Integer,String> pair;
+ if (Tool.class.isAssignableFrom(theClass) && ClusterType.STANDALONE == getClusterType()) {
+ StandaloneClusterControl control = (StandaloneClusterControl) getClusterControl();
+ pair = control.execMapreduceWithStdout(theClass, args);
+ } else {
+ // We're already slurping stdout into memory (not redirecting to file). Might as well add it to error message.
+ pair = getClusterControl().execWithStdout(theClass, args);
+ }
+ Assert.assertEquals("stdout=" + pair.getValue(), 0, pair.getKey().intValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java b/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java
new file mode 100644
index 0000000..b75a74e
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationIT.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+/**
+ * See ACCUMULO-779
+ */
+public class FateStarvationIT extends AccumuloClusterHarness {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void run() throws Exception {
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+
+ c.tableOperations().addSplits(tableName, TestIngest.getSplitPoints(0, 100000, 50));
+
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.random = 89;
+ opts.timestamp = 7;
+ opts.dataSize = 50;
+ opts.rows = 100000;
+ opts.cols = 1;
+ opts.setTableName(tableName);
+ ClientConfiguration clientConf = cluster.getClientConfig();
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConf);
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+
+ c.tableOperations().flush(tableName, null, null, true);
+
+ List<Text> splits = new ArrayList<Text>(TestIngest.getSplitPoints(0, 100000, 67));
+ Random rand = new Random();
+
+ for (int i = 0; i < 100; i++) {
+ int idx1 = rand.nextInt(splits.size() - 1);
+ int idx2 = rand.nextInt(splits.size() - (idx1 + 1)) + idx1 + 1;
+
+ c.tableOperations().compact(tableName, splits.get(idx1), splits.get(idx2), false, false);
+ }
+
+ c.tableOperations().offline(tableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java b/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
new file mode 100644
index 0000000..05d0562
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl.LogWriter;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+
+import com.google.common.collect.Iterators;
+
+public class FunctionalTestUtils {
+
+ public static int countRFiles(Connector c, String tableName) throws Exception {
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ String tableId = c.tableOperations().tableIdMap().get(tableName);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ return Iterators.size(scanner.iterator());
+ }
+
+ static void checkRFiles(Connector c, String tableName, int minTablets, int maxTablets, int minRFiles, int maxRFiles) throws Exception {
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ String tableId = c.tableOperations().tableIdMap().get(tableName);
+ scanner.setRange(new Range(new Text(tableId + ";"), true, new Text(tableId + "<"), true));
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+
+ HashMap<Text,Integer> tabletFileCounts = new HashMap<Text,Integer>();
+
+ for (Entry<Key,Value> entry : scanner) {
+
+ Text row = entry.getKey().getRow();
+
+ Integer count = tabletFileCounts.get(row);
+ if (count == null)
+ count = 0;
+ if (entry.getKey().getColumnFamily().equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
+ count = count + 1;
+ }
+
+ tabletFileCounts.put(row, count);
+ }
+
+ if (tabletFileCounts.size() < minTablets || tabletFileCounts.size() > maxTablets) {
+ throw new Exception("Did not find expected number of tablets " + tabletFileCounts.size());
+ }
+
+ Set<Entry<Text,Integer>> es = tabletFileCounts.entrySet();
+ for (Entry<Text,Integer> entry : es) {
+ if (entry.getValue() > maxRFiles || entry.getValue() < minRFiles) {
+ throw new Exception("tablet " + entry.getKey() + " has " + entry.getValue() + " map files");
+ }
+ }
+ }
+
+ static public void bulkImport(Connector c, FileSystem fs, String table, String dir) throws Exception {
+ String failDir = dir + "_failures";
+ Path failPath = new Path(failDir);
+ fs.delete(failPath, true);
+ fs.mkdirs(failPath);
+
+ // Ensure server can read/modify files
+ FsShell fsShell = new FsShell(fs.getConf());
+ Assert.assertEquals("Failed to chmod " + dir, 0, fsShell.run(new String[] {"-chmod", "-R", "777", dir}));
+ Assert.assertEquals("Failed to chmod " + failDir, 0, fsShell.run(new String[] {"-chmod", "-R", "777", failDir}));
+
+ c.tableOperations().importDirectory(table, dir, failDir, false);
+
+ if (fs.listStatus(failPath).length > 0) {
+ throw new Exception("Some files failed to bulk import");
+ }
+
+ }
+
+ static public void checkSplits(Connector c, String table, int min, int max) throws Exception {
+ Collection<Text> splits = c.tableOperations().listSplits(table);
+ if (splits.size() < min || splits.size() > max) {
+ throw new Exception("# of table splits points out of range, #splits=" + splits.size() + " table=" + table + " min=" + min + " max=" + max);
+ }
+ }
+
+ static public void createRFiles(final Connector c, FileSystem fs, String path, int rows, int splits, int threads) throws Exception {
+ fs.delete(new Path(path), true);
+ ExecutorService threadPool = Executors.newFixedThreadPool(threads);
+ final AtomicBoolean fail = new AtomicBoolean(false);
+ for (int i = 0; i < rows; i += rows / splits) {
+ final TestIngest.Opts opts = new TestIngest.Opts();
+ opts.outputFile = String.format("%s/mf%s", path, i);
+ opts.random = 56;
+ opts.timestamp = 1;
+ opts.dataSize = 50;
+ opts.rows = rows / splits;
+ opts.startRow = i;
+ opts.cols = 1;
+ threadPool.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ } catch (Exception e) {
+ fail.set(true);
+ }
+ }
+ });
+ }
+ threadPool.shutdown();
+ threadPool.awaitTermination(1, TimeUnit.HOURS);
+ assertFalse(fail.get());
+ }
+
+ static public String readAll(InputStream is) throws IOException {
+ byte[] buffer = new byte[4096];
+ StringBuffer result = new StringBuffer();
+ while (true) {
+ int n = is.read(buffer);
+ if (n <= 0)
+ break;
+ result.append(new String(buffer, 0, n));
+ }
+ return result.toString();
+ }
+
+ public static String readAll(MiniAccumuloClusterImpl c, Class<?> klass, Process p) throws Exception {
+ for (LogWriter writer : c.getLogWriters())
+ writer.flush();
+ return readAll(new FileInputStream(c.getConfig().getLogDir() + "/" + klass.getSimpleName() + "_" + p.hashCode() + ".out"));
+ }
+
+ static Mutation nm(String row, String cf, String cq, Value value) {
+ Mutation m = new Mutation(new Text(row));
+ m.put(new Text(cf), new Text(cq), value);
+ return m;
+ }
+
+ static Mutation nm(String row, String cf, String cq, String value) {
+ return nm(row, cf, cq, new Value(value.getBytes()));
+ }
+
+ public static SortedSet<Text> splits(String[] splits) {
+ SortedSet<Text> result = new TreeSet<Text>();
+ for (String split : splits)
+ result.add(new Text(split));
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
new file mode 100644
index 0000000..a73f239
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
+import org.apache.accumulo.gc.SimpleGarbageCollector;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessNotFoundException;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class GarbageCollectorIT extends ConfigurableMacBase {
+ private static final String OUR_SECRET = "itsreallysecret";
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_SECRET, OUR_SECRET);
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_PORT, "0");
+ cfg.setProperty(Property.TSERV_MAXMEM, "5K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ private void killMacGc() throws ProcessNotFoundException, InterruptedException, KeeperException {
+ // kill gc started by MAC
+ getCluster().killProcess(ServerType.GARBAGE_COLLECTOR, getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR).iterator().next());
+ // delete lock in zookeeper if there, this will allow next GC to start quickly
+ String path = ZooUtil.getRoot(new ZooKeeperInstance(getCluster().getClientConfig())) + Constants.ZGC_LOCK;
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ try {
+ ZooLock.deleteLock(zk, path);
+ } catch (IllegalStateException e) {
+
+ }
+
+ assertNull(getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR));
+ }
+
+ @Test
+ public void gcTest() throws Exception {
+ killMacGc();
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
+ TestIngest.Opts opts = new TestIngest.Opts();
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.rows = opts.rows = 10000;
+ vopts.cols = opts.cols = 1;
+ opts.setPrincipal("root");
+ vopts.setPrincipal("root");
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ c.tableOperations().compact("test_ingest", null, null, true, true);
+ int before = countFiles();
+ while (true) {
+ UtilWaitThread.sleep(1000);
+ int more = countFiles();
+ if (more <= before)
+ break;
+ before = more;
+ }
+
+ // restart GC
+ getCluster().start();
+ UtilWaitThread.sleep(15 * 1000);
+ int after = countFiles();
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ assertTrue(after < before);
+ }
+
+ @Test
+ public void gcLotsOfCandidatesIT() throws Exception {
+ killMacGc();
+
+ log.info("Filling metadata table with bogus delete flags");
+ Connector c = getConnector();
+ addEntries(c, new BatchWriterOpts());
+ cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE);
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ UtilWaitThread.sleep(20 * 1000);
+ String output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc);
+ gc.destroy();
+ assertTrue(output.contains("delete candidates has exceeded"));
+ }
+
+ @Test
+ public void dontGCRootLog() throws Exception {
+ killMacGc();
+ // dirty metadata
+ Connector c = getConnector();
+ String table = getUniqueNames(1)[0];
+ c.tableOperations().create(table);
+ // let gc run for a bit
+ cluster.start();
+ UtilWaitThread.sleep(20 * 1000);
+ killMacGc();
+ // kill tservers
+ for (ProcessReference ref : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, ref);
+ }
+ // run recovery
+ cluster.start();
+ // did it recover?
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ Iterators.size(scanner.iterator());
+ }
+
+ private Mutation createDelMutation(String path, String cf, String cq, String val) {
+ Text row = new Text(MetadataSchema.DeletesSection.getRowPrefix() + path);
+ Mutation delFlag = new Mutation(row);
+ delFlag.put(cf, cq, val);
+ return delFlag;
+ }
+
+ @Test
+ public void testInvalidDelete() throws Exception {
+ killMacGc();
+
+ String table = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(table);
+
+ BatchWriter bw2 = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ Mutation m1 = new Mutation("r1");
+ m1.put("cf1", "cq1", "v1");
+ bw2.addMutation(m1);
+ bw2.close();
+
+ getConnector().tableOperations().flush(table, null, null, true);
+
+ // ensure an invalid delete entry does not cause GC to go berserk ACCUMULO-2520
+ getConnector().securityOperations().grantTablePermission(getConnector().whoami(), MetadataTable.NAME, TablePermission.WRITE);
+ BatchWriter bw3 = getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ bw3.addMutation(createDelMutation("", "", "", ""));
+ bw3.addMutation(createDelMutation("", "testDel", "test", "valueTest"));
+ bw3.addMutation(createDelMutation("/", "", "", ""));
+ bw3.close();
+
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ try {
+ String output = "";
+ while (!output.contains("Ingoring invalid deletion candidate")) {
+ UtilWaitThread.sleep(250);
+ try {
+ output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc);
+ } catch (IOException ioe) {
+ log.error("Could not read all from cluster.", ioe);
+ }
+ }
+ } finally {
+ gc.destroy();
+ }
+
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ Assert.assertEquals("r1", entry.getKey().getRow().toString());
+ Assert.assertEquals("cf1", entry.getKey().getColumnFamily().toString());
+ Assert.assertEquals("cq1", entry.getKey().getColumnQualifier().toString());
+ Assert.assertEquals("v1", entry.getValue().toString());
+ Assert.assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testProperPortAdvertisement() throws Exception {
+
+ Connector conn = getConnector();
+ Instance instance = conn.getInstance();
+
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ for (int i = 0; i < 5; i++) {
+ List<String> locks;
+ try {
+ locks = zk.getChildren(path, null);
+ } catch (NoNodeException e) {
+ Thread.sleep(5000);
+ continue;
+ }
+
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
+
+ String lockPath = path + "/" + locks.get(0);
+
+ String gcLoc = new String(zk.getData(lockPath, null));
+
+ Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc, gcLoc.startsWith(Service.GC_CLIENT.name()));
+ int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
+ Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc, -1, loc);
+ String addr = gcLoc.substring(loc + 1);
+
+ int addrSplit = addr.indexOf(':');
+ Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1, addrSplit);
+
+ String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
+ // We shouldn't have the "bindall" address in zk
+ Assert.assertNotEquals("0.0.0.0", host);
+ // Nor should we have the "random port" in zk
+ Assert.assertNotEquals(0, Integer.parseInt(port));
+ return;
+ }
+
+ Thread.sleep(5000);
+ }
+
+ Assert.fail("Could not find advertised GC address");
+ }
+
+ private int countFiles() throws Exception {
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ Path path = new Path(cluster.getConfig().getDir() + "/accumulo/tables/1/*/*.rf");
+ return Iterators.size(Arrays.asList(fs.globStatus(path)).iterator());
+ }
+
+ public static void addEntries(Connector conn, BatchWriterOpts bwOpts) throws Exception {
+ conn.securityOperations().grantTablePermission(conn.whoami(), MetadataTable.NAME, TablePermission.WRITE);
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, bwOpts.getBatchWriterConfig());
+
+ for (int i = 0; i < 100000; ++i) {
+ final Text emptyText = new Text("");
+ Text row = new Text(String.format("%s/%020d/%s", MetadataSchema.DeletesSection.getRowPrefix(), i,
+ "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj"));
+ Mutation delFlag = new Mutation(row);
+ delFlag.put(emptyText, emptyText, new Value(new byte[] {}));
+ bw.addMutation(delFlag);
+ }
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
new file mode 100644
index 0000000..59d8259
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.start.Main;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+public class HalfDeadTServerIT extends ConfigurableMacBase {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.GENERAL_RPC_TIMEOUT, "5s");
+ cfg.useMiniDFS(true);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ class DumpOutput extends Daemon {
+
+ private final BufferedReader rdr;
+ private final StringBuilder output;
+
+ DumpOutput(InputStream is) {
+ rdr = new BufferedReader(new InputStreamReader(is));
+ output = new StringBuilder();
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ String line = rdr.readLine();
+ if (line == null)
+ break;
+ System.out.println(line);
+ output.append(line);
+ output.append("\n");
+ }
+ } catch (IOException ex) {
+ log.error("IOException", ex);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return output.toString();
+ }
+ }
+
+ @Test
+ public void testRecover() throws Exception {
+ test(10);
+ }
+
+ @Test
+ public void testTimeout() throws Exception {
+ String results = test(20, true);
+ if (results != null) {
+ if (!results.contains("Session expired")) {
+ log.info("Failed to find 'Session expired' in output, but TServer did die which is expected");
+ }
+ }
+ }
+
+ public String test(int seconds) throws Exception {
+ return test(seconds, false);
+ }
+
+ public String test(int seconds, boolean expectTserverDied) throws Exception {
+ if (!makeDiskFailureLibrary())
+ return null;
+ Connector c = getConnector();
+ assertEquals(1, c.instanceOperations().getTabletServers().size());
+
+ // create our own tablet server with the special test library
+ String javaHome = System.getProperty("java.home");
+ String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
+ String classpath = System.getProperty("java.class.path");
+ classpath = new File(cluster.getConfig().getDir(), "conf") + File.pathSeparator + classpath;
+ String className = TabletServer.class.getName();
+ ArrayList<String> argList = new ArrayList<String>();
+ argList.addAll(Arrays.asList(javaBin, "-cp", classpath));
+ argList.addAll(Arrays.asList(Main.class.getName(), className));
+ ProcessBuilder builder = new ProcessBuilder(argList);
+ Map<String,String> env = builder.environment();
+ env.put("ACCUMULO_HOME", cluster.getConfig().getDir().getAbsolutePath());
+ env.put("ACCUMULO_LOG_DIR", cluster.getConfig().getLogDir().getAbsolutePath());
+ String trickFilename = cluster.getConfig().getLogDir().getAbsolutePath() + "/TRICK_FILE";
+ env.put("TRICK_FILE", trickFilename);
+ String libPath = System.getProperty("user.dir") + "/target/fake_disk_failure.so";
+ env.put("LD_PRELOAD", libPath);
+ env.put("DYLD_INSERT_LIBRARIES", libPath);
+ env.put("DYLD_FORCE_FLAT_NAMESPACE", "true");
+ Process ingest = null;
+ Process tserver = builder.start();
+ DumpOutput t = new DumpOutput(tserver.getInputStream());
+ try {
+ t.start();
+ UtilWaitThread.sleep(1000);
+ // don't need the regular tablet server
+ cluster.killProcess(ServerType.TABLET_SERVER, cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next());
+ UtilWaitThread.sleep(1000);
+ c.tableOperations().create("test_ingest");
+ assertEquals(1, c.instanceOperations().getTabletServers().size());
+ int rows = 100 * 1000;
+ ingest = cluster.exec(TestIngest.class, "-u", "root", "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "-p", ROOT_PASSWORD, "--rows", rows
+ + "");
+ UtilWaitThread.sleep(500);
+
+ // block I/O with some side-channel trickiness
+ File trickFile = new File(trickFilename);
+ try {
+ assertTrue(trickFile.createNewFile());
+ UtilWaitThread.sleep(seconds * 1000);
+ } finally {
+ if (!trickFile.delete()) {
+ log.error("Couldn't delete " + trickFile);
+ }
+ }
+
+ if (seconds <= 10) {
+ assertEquals(0, ingest.waitFor());
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.rows = rows;
+ vopts.setPrincipal("root");
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ } else {
+ UtilWaitThread.sleep(5 * 1000);
+ tserver.waitFor();
+ t.join();
+ tserver = null;
+ }
+ // verify the process was blocked
+ String results = t.toString();
+ assertTrue(results.contains("sleeping\nsleeping\nsleeping\n"));
+ return results;
+ } finally {
+ if (ingest != null) {
+ ingest.destroy();
+ ingest.waitFor();
+ }
+ if (tserver != null) {
+ try {
+ if (expectTserverDied) {
+ try {
+ tserver.exitValue();
+ } catch (IllegalThreadStateException e) {
+ fail("Expected TServer to kill itself, but it is still running");
+ }
+ }
+ } finally {
+ tserver.destroy();
+ tserver.waitFor();
+ t.join();
+ }
+ }
+ }
+ }
+
+ private boolean makeDiskFailureLibrary() throws Exception {
+ String root = System.getProperty("user.dir");
+ String source = root + "/src/test/c/fake_disk_failure.c";
+ String lib = root + "/target/fake_disk_failure.so";
+ String platform = System.getProperty("os.name");
+ String cmd[];
+ if (platform.equals("Darwin")) {
+ cmd = new String[] {"gcc", "-arch", "x86_64", "-arch", "i386", "-dynamiclib", "-O3", "-fPIC", source, "-o", lib};
+ } else {
+ cmd = new String[] {"gcc", "-D_GNU_SOURCE", "-Wall", "-fPIC", source, "-shared", "-o", lib, "-ldl"};
+ }
+ Process gcc = Runtime.getRuntime().exec(cmd);
+ return gcc.waitFor() == 0;
+ }
+
+}