You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/07/17 04:33:18 UTC
[16/50] [abbrv] git commit: ACCUMULO-1537 started the stress tests,
fixed some tests, made others more reliable
ACCUMULO-1537 started the stress tests, fixed some tests, made others more reliable
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1499562 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/390ca3f2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/390ca3f2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/390ca3f2
Branch: refs/heads/ACCUMULO-1496
Commit: 390ca3f2ebe75585ed8c97512e9985711f5d641d
Parents: 446a37a
Author: Eric C. Newton <ec...@apache.org>
Authored: Wed Jul 3 21:21:48 2013 +0000
Committer: Eric C. Newton <ec...@apache.org>
Committed: Wed Jul 3 21:21:48 2013 +0000
----------------------------------------------------------------------
.../accumulo/test/TestMultiTableIngest.java | 4 +-
.../test/functional/CreateManyScannersTest.java | 51 ---------
.../test/functional/BigRootTabletIT.java | 63 +++++++++++
.../accumulo/test/functional/BinaryIT.java | 2 +-
.../test/functional/BinaryStressIT.java | 55 ++++++++++
.../accumulo/test/functional/BloomFilterIT.java | 20 ++--
.../test/functional/ChaoticBlancerIT.java | 65 +++++++++++
.../test/functional/CreateManyScannersIT.java | 34 ++++++
.../test/functional/FunctionalTestUtils.java | 8 ++
.../test/functional/MetadataMaxFiles.java | 108 +++++++++++++++++++
.../accumulo/test/functional/ReadWriteIT.java | 2 +-
.../accumulo/test/functional/SplitIT.java | 8 +-
.../accumulo/test/functional/TimeoutIT.java | 2 +-
.../accumulo/test/functional/WriteLotsIT.java | 66 ++++++++++++
test/system/auto/stress/batchWrite.py | 77 -------------
test/system/auto/stress/bigRootTablet.py | 50 ---------
test/system/auto/stress/binary.py | 52 ---------
test/system/auto/stress/manyScanners.py | 31 ------
test/system/auto/stress/metadataMaxFiles.py | 66 ------------
19 files changed, 419 insertions(+), 345 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
index b051988..1458065 100644
--- a/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestMultiTableIngest.java
@@ -54,7 +54,7 @@ public class TestMultiTableIngest {
scanner.setBatchSize(scanOpts.scanBatchSize);
int count = i;
for (Entry<Key,Value> elt : scanner) {
- String expected = String.format("%05d", count);
+ String expected = String.format("%06d", count);
if (!elt.getKey().getRow().toString().equals(expected))
throw new RuntimeException("entry " + elt + " does not match expected " + expected + " in table " + table);
count += tableNames.size();
@@ -96,7 +96,7 @@ public class TestMultiTableIngest {
// populate
for (int i = 0; i < opts.count; i++) {
- Mutation m = new Mutation(new Text(String.format("%05d", i)));
+ Mutation m = new Mutation(new Text(String.format("%06d", i)));
m.put(new Text("col" + Integer.toString((i % 3) + 1)), new Text("qual"), new Value("junk".getBytes()));
b.getBatchWriter(tableNames.get(i % tableNames.size())).addMutation(m);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java b/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
deleted file mode 100644
index c84fb9b..0000000
--- a/test/src/main/java/org/apache/accumulo/test/functional/CreateManyScannersTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.security.Authorizations;
-
-public class CreateManyScannersTest extends FunctionalTest {
-
- @Override
- public void cleanup() throws Exception {
-
- }
-
- @Override
- public Map<String,String> getInitialConfig() {
- return Collections.emptyMap();
- }
-
- @Override
- public List<TableSetup> getTablesToCreate() {
- return Collections.singletonList(new TableSetup("mscant"));
- }
-
- @Override
- public void run() throws Exception {
- Connector connector = getConnector();
- for (int i = 0; i < 100000; i++) {
- connector.createScanner("mscant", Authorizations.EMPTY);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java
new file mode 100644
index 0000000..308560f
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BigRootTabletIT.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.RootTable;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.junit.Test;
+
+public class BigRootTabletIT extends MacTest {
+ // ACCUMULO-542: A large root tablet will fail to load if it does't fit in the tserver scan buffers
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String,String> siteConfig = new HashMap<String, String>();
+ siteConfig.put(Property.TABLE_SCAN_MAXMEM.getKey(), "1024");
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "60m");
+ cfg.setSiteConfig(siteConfig );
+ }
+
+ @Test(timeout=60*1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().addSplits(MetadataTable.NAME, FunctionalTestUtils.splits("0 1 2 3 4 5 6 7 8 9 a".split(" ")));
+ for (int i = 0; i < 10; i++) {
+ c.tableOperations().create("" + i);
+ c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ c.tableOperations().flush(RootTable.NAME, null, null, true);
+ }
+ cluster.stop();
+ cluster.start();
+ int count = 0;
+ for (@SuppressWarnings("unused") Entry<Key,Value> entry : c.createScanner(RootTable.NAME, Authorizations.EMPTY))
+ count++;
+ assertTrue(count > 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
index 484c55c..fb28715 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BinaryIT.java
@@ -45,7 +45,7 @@ public class BinaryIT extends MacTest {
runTest(c);
}
- void runTest(Connector c) throws Exception {
+ public static void runTest(Connector c) throws Exception {
BatchWriterOpts bwOpts = new BatchWriterOpts();
ScannerOpts scanOpts = new ScannerOpts();
TestBinaryRows.Opts opts = new TestBinaryRows.Opts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
new file mode 100644
index 0000000..8c3af67
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+public class BinaryStressIT extends MacTest {
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String,String> siteConfig = new HashMap<String,String>();
+ siteConfig.put(Property.TSERV_MAXMEM.getKey(), "50K");
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+ cfg.setSiteConfig(siteConfig );
+ }
+
+ @Test(timeout=60*1000)
+ public void binaryStressTest() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("bt");
+ c.tableOperations().setProperty("bt", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+ BinaryIT.runTest(c);
+ String id = c.tableOperations().tableIdMap().get("bt");
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ FileStatus[] dir = fs.listStatus(new Path(cluster.getConfig().getDir() + "/accumulo/tables/" + id));
+ assertTrue(dir.length > 7);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 19153df..d222991 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -49,15 +49,15 @@ public class BloomFilterIT extends MacTest {
cfg.setDefaultMemory(500, MemoryUnit.MEGABYTE);
}
- @Test(timeout=120*1000)
+ @Test(timeout=500*1000)
public void test() throws Exception {
Connector c = getConnector();
for (String table : "bt1 bt2 bt3 bt4".split(" ")) {
c.tableOperations().create(table);
}
- write(c, "bt1", 1, 0, 1000000000, 500);
- write(c, "bt2", 2, 0, 1000000000, 500);
- write(c, "bt3", 3, 0, 1000000000, 500);
+ write(c, "bt1", 1, 0, 1000000000, 100);
+ write(c, "bt2", 2, 0, 1000000000, 100);
+ write(c, "bt3", 3, 0, 1000000000, 100);
// test inserting an empty key
BatchWriter bw = c.createBatchWriter("bt4", new BatchWriterConfig());
@@ -80,9 +80,9 @@ public class BloomFilterIT extends MacTest {
FunctionalTestUtils.checkRFiles(c, "bt4", 1, 1, 1, 1);
// these queries should only run quickly if bloom filters are working, so lets get a base
- long t1 = query(c, "bt1", 1, 0, 1000000000, 100000, 500);
- long t2 = query(c, "bt2", 2, 0, 1000000000, 100000, 500);
- long t3 = query(c, "bt3", 3, 0, 1000000000, 100000, 500);
+ long t1 = query(c, "bt1", 1, 0, 1000000000, 100000, 100);
+ long t2 = query(c, "bt2", 2, 0, 1000000000, 100000, 100);
+ long t3 = query(c, "bt3", 3, 0, 1000000000, 100000, 100);
c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
@@ -102,9 +102,9 @@ public class BloomFilterIT extends MacTest {
// these queries should only run quickly if bloom
// filters are working
- long tb1 = query(c, "bt1", 1, 0, 1000000000, 100000, 500);
- long tb2 = query(c, "bt2", 2, 0, 1000000000, 100000, 500);
- long tb3 = query(c, "bt3", 3, 0, 1000000000, 100000, 500);
+ long tb1 = query(c, "bt1", 1, 0, 1000000000, 100000, 100);
+ long tb2 = query(c, "bt2", 2, 0, 1000000000, 100000, 100);
+ long tb3 = query(c, "bt3", 3, 0, 1000000000, 100000, 100);
timeCheck(t1, tb1);
timeCheck(t2, tb2);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
new file mode 100644
index 0000000..fb46f1e
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ChaoticBlancerIT.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.master.balancer.ChaoticLoadBalancer;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ChaoticBlancerIT extends MacTest {
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String,String> siteConfig = new HashMap<String, String>();
+ siteConfig.put(Property.TSERV_MAXMEM.getKey(), "10K");
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
+ siteConfig.put(Property.TABLE_LOAD_BALANCER.getKey(), ChaoticLoadBalancer.class.getName());
+ cfg.setSiteConfig(siteConfig );
+ }
+
+ @Test(timeout=120*1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 200; i++) {
+ splits.add(new Text(String.format("%03d", i)));
+ }
+ c.tableOperations().create("unused");
+ c.tableOperations().addSplits("unused", splits);
+ TestIngest.Opts opts = new TestIngest.Opts();
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.rows = opts.rows = 200000;
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ c.tableOperations().flush("test_ingest", null, null, true);
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java b/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
new file mode 100644
index 0000000..c41fae1
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CreateManyScannersIT.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.Test;
+
+public class CreateManyScannersIT extends MacTest {
+
+ @Test(timeout=10*1000)
+ public void run() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("mscant");
+ for (int i = 0; i < 100000; i++) {
+ c.createScanner("mscant", Authorizations.EMPTY);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java b/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
index 4af2c96..8fd50f5 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
@@ -23,8 +23,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashMap;
+import java.util.SortedSet;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -160,5 +162,11 @@ public class FunctionalTestUtils {
static Mutation nm(String row, String cf, String cq, String value) {
return nm(row, cf, cq, new Value(value.getBytes()));
}
+ public static SortedSet<Text> splits(String [] splits) {
+ SortedSet<Text> result = new TreeSet<Text>();
+ for (String split : splits)
+ result.add(new Text(split));
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
new file mode 100644
index 0000000..ae503de
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataMaxFiles.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.RootTable;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class MetadataMaxFiles extends MacTest {
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String,String> siteConfig = new HashMap<String, String>();
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
+ siteConfig.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "10");
+ cfg.setSiteConfig(siteConfig );
+ }
+
+ @Test(timeout=240*1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 1000; i++) {
+ splits.add(new Text(String.format("%03d", i)));
+ }
+ c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10000");
+ for (int i = 0; i < 5; i++) {
+ String tableName = "table" + i;
+ log.info("Creating " + tableName);
+ c.tableOperations().create(tableName);
+ log.info("adding splits");
+ c.tableOperations().addSplits(tableName, splits);
+ log.info("flushing");
+ c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ c.tableOperations().flush(RootTable.NAME, null, null, true);
+ }
+ UtilWaitThread.sleep(20*1000);
+ log.info("shutting down");
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+ log.info("starting up");
+ cluster.start();
+
+ UtilWaitThread.sleep(30*1000);
+
+ while (true) {
+ MasterMonitorInfo stats = null;
+ TCredentials creds = CredentialHelper.create("root", new PasswordToken(MacTest.PASSWORD), c.getInstance().getInstanceName());
+ Client client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(c.getInstance());
+ stats = client.getMasterStats(Tracer.traceInfo(), creds);
+ } finally {
+ if (client != null)
+ MasterClient.close(client);
+ }
+ int tablets = 0;
+ for (TabletServerStatus tserver : stats.tServerInfo) {
+ for (Entry<String,TableInfo> entry : tserver.tableMap.entrySet()) {
+ if (entry.getKey().startsWith("!"))
+ continue;
+ tablets += entry.getValue().onlineTablets;
+ }
+ }
+ if (tablets == 5005)
+ break;
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index bc89e20..7c74326 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@ -57,7 +57,7 @@ import org.junit.Test;
public class ReadWriteIT extends MacTest {
- static final int ROWS = 20000;
+ static final int ROWS = 200000;
static final int COLS = 1;
static final String COLF = "colf";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
index 741e216..c4719e0 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java
@@ -48,7 +48,7 @@ public class SplitIT extends MacTest {
public void configure(MiniAccumuloConfig cfg) {
Map<String,String> siteConfig = new HashMap<String,String>();
siteConfig.put(Property.TSERV_MAXMEM.getKey(), "5K");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1s");
+ siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
cfg.setSiteConfig(siteConfig);
}
@@ -92,8 +92,8 @@ public class SplitIT extends MacTest {
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
c.tableOperations().setProperty("test_ingest", Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
ReadWriteIT.interleaveTest(c);
- UtilWaitThread.sleep(5 * 1000);
- assertTrue(c.tableOperations().listSplits("test_ingest").size() > 10);
+ UtilWaitThread.sleep(5*1000);
+ assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
}
@Test(timeout = 120 * 1000)
@@ -102,6 +102,8 @@ public class SplitIT extends MacTest {
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
DeleteIT.deleteTest(c);
+ c.tableOperations().flush("test_ingest", null, null, true);
+ UtilWaitThread.sleep(5*1000);
assertTrue(c.tableOperations().listSplits("test_ingest").size() > 30);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java b/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
index 2dc6802..2c3c86d 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
@@ -42,7 +42,7 @@ import org.junit.Test;
*/
public class TimeoutIT extends MacTest {
- @Test
+ @Test(timeout=30*1000)
public void run() throws Exception {
Connector conn = getConnector();
testBatchWriterTimeout(conn);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java b/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
new file mode 100644
index 0000000..df8e656
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.junit.Test;
+
+public class WriteLotsIT extends MacTest {
+
+ @Test(timeout=20*1000)
+ public void writeLots() throws Exception {
+ final Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ final AtomicReference<Exception> ref = new AtomicReference<Exception>();
+ List<Thread> threads = new ArrayList<Thread>();
+ for (int i = 0; i < 10; i++) {
+ final int index = i;
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.startRow = index * 10000;
+ opts.rows = 10000;
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ } catch (Exception ex) {
+ ref.set(ex);
+ }
+ }
+ };
+ t.start();
+ threads.add(t);
+ }
+ for (Thread thread : threads) {
+ thread.join();
+ }
+ if (ref.get() != null) {
+ throw ref.get();
+ }
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.rows = 10000 * 10;
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/batchWrite.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/batchWrite.py b/test/system/auto/stress/batchWrite.py
deleted file mode 100755
index f2f83b0..0000000
--- a/test/system/auto/stress/batchWrite.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import logging
-import unittest
-import time
-
-from TestUtils import TestUtilsMixin, ROOT_PASSWORD
-
-log = logging.getLogger('test.auto')
-
-class WriteLots(unittest.TestCase, TestUtilsMixin):
- """Start a clean accumulo, ingest some data using lots of clients at once,
- and verify it was stored properly"""
-
- order = 30
- settings = TestUtilsMixin.settings.copy()
- settings['instance.zookeeper.timeout'] = '40s'
-
- def ingest(self, host, start, count, **kwargs):
- klass = 'org.apache.accumulo.test.TestIngest'
- args = '--user root --size 50 --random 56 --rows %d --start %d --cols 1 -p %s' % (count, start, ROOT_PASSWORD)
- return self.runClassOn(host, klass, args.split(), **kwargs)
-
- def setUp(self):
- TestUtilsMixin.setUp(self);
-
- # initialize the database
- self.createTable("test_ingest")
-
- def tearDown(self):
- TestUtilsMixin.tearDown(self)
- self.pkill(self.masterHost(), 'TestIngest')
-
- def runTest(self):
- N = 10*len(self.hosts)
- waitTime = 60 * N * self.options.rows / 200000 + 90
-
- log.info("Starting %d clients", N)
- handles = []
- for i in range(N):
- # start test ingestion
- handle = self.ingest(self.hosts[i % len(self.hosts)],
- i * self.options.rows,
- self.options.rows)
- handles.append(handle)
-
- end = time.time() + waitTime
- for handle in handles:
- waitTime = end - time.time()
- log.debug("Waiting %s seconds", waitTime)
- self.waitForStop(handle, waitTime)
-
- log.info("Verifying Ingestion")
- self.waitForStop(self.verify(self.masterHost(), self.options.rows * N),
- waitTime)
- self.shutdown_accumulo()
-
-
-def suite():
- result = unittest.TestSuite()
- result.addTest(WriteLots())
- return result
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/bigRootTablet.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/bigRootTablet.py b/test/system/auto/stress/bigRootTablet.py
deleted file mode 100755
index d69a79c..0000000
--- a/test/system/auto/stress/bigRootTablet.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import logging
-import unittest
-
-from TestUtils import TestUtilsMixin
-
-log = logging.getLogger('test.auto')
-
-class BigRootTablet(TestUtilsMixin, unittest.TestCase):
- "ACCUMULO-542: A large root tablet will fail to load if it does't fit in the tserver scan buffers"
-
- order = 80
-
- settings = TestUtilsMixin.settings.copy()
- settings['table.scan.max.memory'] = '1024'
- settings['tserver.compaction.major.delay'] = '60m'
-
- def setUp(self):
- TestUtilsMixin.setUp(self);
-
- def tearDown(self):
- TestUtilsMixin.tearDown(self);
-
- def runTest(self):
- cmd = 'table !METADATA\naddsplits 0 1 2 3 4 5 6 7 8 9 a\n'
- for i in range(10):
- cmd += 'createtable %s\nflush -t !METADATA\n' % i
- self.shell(self.masterHost(), cmd)
- self.stop_accumulo()
- self.start_accumulo()
-
-def suite():
- result = unittest.TestSuite()
- result.addTest(BigRootTablet())
- return result
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/binary.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/binary.py b/test/system/auto/stress/binary.py
deleted file mode 100755
index 41f584d..0000000
--- a/test/system/auto/stress/binary.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-import os
-import logging
-
-from TestUtils import ACCUMULO_DIR
-from simple.binary import BinaryTest
-
-log = logging.getLogger('test.auto')
-
-class BinaryStressTest(BinaryTest) :
- order = 80
-
- tableSettings = BinaryTest.tableSettings.copy()
- tableSettings['bt'] = {
- 'table.split.threshold': '10K',
- }
- settings = BinaryTest.settings.copy()
- settings.update({
- 'tserver.memory.maps.max':'50K',
- 'tserver.compaction.major.delay': 0,
- })
-
- def runTest(self):
- BinaryTest.runTest(self)
- handle = self.runOn(self.masterHost(), [
- 'hadoop', 'fs', '-ls', os.path.join(ACCUMULO_DIR,'tables',self.getTableId('bt'))
- ])
- out, err = handle.communicate()
- if len(out.split('\n')) < 8:
- log.debug(out)
- self.assert_(len(out.split('\n')) > 7)
-
-def suite():
- result = unittest.TestSuite()
- result.addTest(BinaryStressTest())
- return result
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/manyScanners.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/manyScanners.py b/test/system/auto/stress/manyScanners.py
deleted file mode 100755
index 229b931..0000000
--- a/test/system/auto/stress/manyScanners.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from JavaTest import JavaTest
-
-import unittest
-
-class CreateManyScannersTest(JavaTest):
- "Test creating a lot of scanners"
-
- order = 9999
- testClass="org.apache.accumulo.test.functional.CreateManyScannersTest"
- maxRuntime = 60
-
-
-def suite():
- result = unittest.TestSuite()
- result.addTest(CreateManyScannersTest())
- return result
http://git-wip-us.apache.org/repos/asf/accumulo/blob/390ca3f2/test/system/auto/stress/metadataMaxFiles.py
----------------------------------------------------------------------
diff --git a/test/system/auto/stress/metadataMaxFiles.py b/test/system/auto/stress/metadataMaxFiles.py
deleted file mode 100755
index 8a65b81..0000000
--- a/test/system/auto/stress/metadataMaxFiles.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-from TestUtils import TestUtilsMixin
-import os
-
-import logging
-log = logging.getLogger('test.auto')
-
-class MetadataMaxFiles(TestUtilsMixin, unittest.TestCase):
- "open a large !METADATA with too few files"
-
- order=75
-
- settings = TestUtilsMixin.settings.copy()
- settings['tserver.compaction.major.delay'] = 1
-
- def runTest(self):
- # Create a bunch of tables with splits to split the !METADATA table
- self.splitfile = 'splits'
- fp = open(self.splitfile, 'w')
- for i in range(1000):
- fp.write('%03d\n' % i)
- fp.close()
- self.splitfile = os.path.realpath(self.splitfile)
- self.shell(self.masterHost(),
- 'config -t !METADATA -s table.split.threshold=10000\n' +
- ''.join(['createtable test%d -sf %s\nflush -t !METADATA\n' % (i, self.splitfile) for i in range(5)]))
- self.shutdown_accumulo(150)
-
- # reconfigure accumulo to use a very small number of files
- self.stop_accumulo()
- self.settings['tserver.scan.files.open.max'] = 10
- self.create_config_file(self.settings)
-
- # make sure the master knows about all the tables we created
- self.sleep(5)
- self.start_accumulo()
- self.sleep(60)
- h = self.runOn(self.masterHost(),
- [self.accumulo_sh(), 'org.apache.accumulo.test.GetMasterStats'])
- out, err = h.communicate()
- self.assert_(len([x for x in out.split('\n') if x.find(' Tablets: 1001') == 0]) == 5)
-
- def tearDown(self):
- TestUtilsMixin.tearDown(self)
- os.unlink(self.splitfile)
-
-def suite():
- result = unittest.TestSuite()
- result.addTest(MetadataMaxFiles())
- return result