You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2016/04/03 05:19:04 UTC
[1/3] accumulo git commit: ACCUMULO-4175 Validate the location of
test ITs
Repository: accumulo
Updated Branches:
refs/heads/master 574234293 -> ada31a9b8
ACCUMULO-4175 Validate the location of test ITs
* Verify ITs exist only in src/main/java for test module
(change due to ACCUMULO-3871)
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f2840273
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f2840273
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f2840273
Branch: refs/heads/master
Commit: f284027334c159e806f94e881e8648b4c0ce6dd9
Parents: 5742342
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Apr 1 13:25:41 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Apr 1 14:57:09 2016 -0400
----------------------------------------------------------------------
test/pom.xml | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f2840273/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index e78a9c1..6da0f0e 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -259,6 +259,28 @@
</plugin>
</plugins>
</pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>check-for-misplaced-ITs</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>validate</phase>
+ <configuration>
+ <executable>bash</executable>
+ <arguments>
+ <argument>-c</argument>
+ <argument>! find src/test/java -name '*IT.java' -exec echo '[ERROR] {} should be in src/main/java' \; | grep 'src/test/java'</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
</build>
<profiles>
<profile>
[3/3] accumulo git commit: Merge remote-tracking branch
'ctubbsii/ACCUMULO-4175'
Posted by ct...@apache.org.
Merge remote-tracking branch 'ctubbsii/ACCUMULO-4175'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ada31a9b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ada31a9b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ada31a9b
Branch: refs/heads/master
Commit: ada31a9b8c87f817c5f5ce7135e369ef636dee2c
Parents: 5742342 7cefd87
Author: Christopher Tubbs <ct...@apache.org>
Authored: Sat Apr 2 23:17:02 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Sat Apr 2 23:17:02 2016 -0400
----------------------------------------------------------------------
test/pom.xml | 22 +++
.../test/functional/SessionBlockVerifyIT.java | 176 +++++++++++++++++++
.../test/functional/SessionBlockVerifyIT.java | 176 -------------------
3 files changed, 198 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
[2/3] accumulo git commit: ACCUMULO-4175 Move ITs to correct directory
Posted by ct...@apache.org.
ACCUMULO-4175 Move ITs to correct directory
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7cefd872
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7cefd872
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7cefd872
Branch: refs/heads/master
Commit: 7cefd872f9276ebb3a4be4cfcdda8f3112ccd250
Parents: f284027
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Apr 1 14:55:44 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Apr 1 14:57:10 2016 -0400
----------------------------------------------------------------------
.../test/functional/SessionBlockVerifyIT.java | 176 +++++++++++++++++++
.../test/functional/SessionBlockVerifyIT.java | 176 -------------------
2 files changed, 176 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7cefd872/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java
new file mode 100644
index 0000000..05f304b
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * Verify that we have resolved blocking issue by ensuring that we have not lost scan sessions which we know to currently be running
+ */
+public class SessionBlockVerifyIT extends ScanSessionTimeOutIT {
+ private static final Logger log = LoggerFactory.getLogger(SessionBlockVerifyIT.class);
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ Map<String,String> siteConfig = cfg.getSiteConfig();
+ cfg.setNumTservers(1);
+ siteConfig.put(Property.TSERV_SESSION_MAXIDLE.getKey(), getMaxIdleTimeString());
+ siteConfig.put(Property.TSERV_READ_AHEAD_MAXCONCURRENT.getKey(), "11");
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ protected String getMaxIdleTimeString() {
+ return "1s";
+ }
+
+ ExecutorService service = Executors.newFixedThreadPool(10);
+
+ @Test
+ public void run() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+
+ for (int i = 0; i < 1000; i++) {
+ Mutation m = new Mutation(new Text(String.format("%08d", i)));
+ for (int j = 0; j < 3; j++)
+ m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
+
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ Scanner scanner = c.createScanner(tableName, new Authorizations());
+ scanner.setReadaheadThreshold(20000);
+ scanner.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000)));
+
+ // test by making a slow iterator and then a couple of fast ones.
+ // when then checking we shouldn't have any running except the slow iterator
+ IteratorSetting setting = new IteratorSetting(21, SlowIterator.class);
+ SlowIterator.setSeekSleepTime(setting, Long.MAX_VALUE);
+ SlowIterator.setSleepTime(setting, Long.MAX_VALUE);
+ scanner.addScanIterator(setting);
+
+ final Iterator<Entry<Key,Value>> slow = scanner.iterator();
+
+ final List<Future<Boolean>> callables = new ArrayList<Future<Boolean>>();
+ final CountDownLatch latch = new CountDownLatch(10);
+ for (int i = 0; i < 10; i++) {
+ Future<Boolean> callable = service.submit(new Callable<Boolean>() {
+ public Boolean call() {
+ latch.countDown();
+ while (slow.hasNext()) {
+
+ slow.next();
+ }
+ return slow.hasNext();
+ }
+ });
+ callables.add(callable);
+ }
+
+ latch.await();
+
+ log.info("Starting SessionBlockVerifyIT");
+
+ // let's add more for good measure.
+ for (int i = 0; i < 2; i++) {
+ Scanner scanner2 = c.createScanner(tableName, new Authorizations());
+
+ scanner2.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000)));
+
+ scanner2.setBatchSize(1);
+ Iterator<Entry<Key,Value>> iter = scanner2.iterator();
+ // call super's verify mechanism
+ verify(iter, 0, 1000);
+
+ }
+
+ int sessionsFound = 0;
+ // we have configured 1 tserver, so we can grab the one and only
+ String tserver = Iterables.getOnlyElement(c.instanceOperations().getTabletServers());
+
+ final List<ActiveScan> scans = c.instanceOperations().getActiveScans(tserver);
+
+ for (ActiveScan scan : scans) {
+ // only here to minimize chance of seeing meta extent scans
+
+ if (tableName.equals(scan.getTable()) && scan.getSsiList().size() > 0) {
+ assertEquals("Not the expected iterator", 1, scan.getSsiList().size());
+ assertTrue("Not the expected iterator", scan.getSsiList().iterator().next().contains("SlowIterator"));
+ sessionsFound++;
+ }
+
+ }
+
+ /**
+ * The message below indicates the problem that we experience within ACCUMULO-3509. The issue manifests as a blockage in the Scanner synchronization that
+ * prevent us from making the close call against it. Since the close blocks until a read is finished, we ultimately have a block within the sweep of
+ * SessionManager. As a result never reap subsequent idle sessions AND we will orphan the sessionsToCleanup in the sweep, leading to an inaccurate count
+ * within sessionsFound.
+ */
+ assertEquals("Must have ten sessions. Failure indicates a synchronization block within the sweep mechanism", 10, sessionsFound);
+ for (Future<Boolean> callable : callables) {
+ callable.cancel(true);
+ }
+ service.shutdown();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7cefd872/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java
deleted file mode 100644
index 05f304b..0000000
--- a/test/src/test/java/org/apache/accumulo/test/functional/SessionBlockVerifyIT.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.admin.ActiveScan;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Iterables;
-
-/**
- * Verify that we have resolved blocking issue by ensuring that we have not lost scan sessions which we know to currently be running
- */
-public class SessionBlockVerifyIT extends ScanSessionTimeOutIT {
- private static final Logger log = LoggerFactory.getLogger(SessionBlockVerifyIT.class);
-
- @Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> siteConfig = cfg.getSiteConfig();
- cfg.setNumTservers(1);
- siteConfig.put(Property.TSERV_SESSION_MAXIDLE.getKey(), getMaxIdleTimeString());
- siteConfig.put(Property.TSERV_READ_AHEAD_MAXCONCURRENT.getKey(), "11");
- cfg.setSiteConfig(siteConfig);
- }
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 60;
- }
-
- @Override
- protected String getMaxIdleTimeString() {
- return "1s";
- }
-
- ExecutorService service = Executors.newFixedThreadPool(10);
-
- @Test
- public void run() throws Exception {
- Connector c = getConnector();
- String tableName = getUniqueNames(1)[0];
- c.tableOperations().create(tableName);
-
- BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
-
- for (int i = 0; i < 1000; i++) {
- Mutation m = new Mutation(new Text(String.format("%08d", i)));
- for (int j = 0; j < 3; j++)
- m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8)));
-
- bw.addMutation(m);
- }
-
- bw.close();
-
- Scanner scanner = c.createScanner(tableName, new Authorizations());
- scanner.setReadaheadThreshold(20000);
- scanner.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000)));
-
- // test by making a slow iterator and then a couple of fast ones.
- // when then checking we shouldn't have any running except the slow iterator
- IteratorSetting setting = new IteratorSetting(21, SlowIterator.class);
- SlowIterator.setSeekSleepTime(setting, Long.MAX_VALUE);
- SlowIterator.setSleepTime(setting, Long.MAX_VALUE);
- scanner.addScanIterator(setting);
-
- final Iterator<Entry<Key,Value>> slow = scanner.iterator();
-
- final List<Future<Boolean>> callables = new ArrayList<Future<Boolean>>();
- final CountDownLatch latch = new CountDownLatch(10);
- for (int i = 0; i < 10; i++) {
- Future<Boolean> callable = service.submit(new Callable<Boolean>() {
- public Boolean call() {
- latch.countDown();
- while (slow.hasNext()) {
-
- slow.next();
- }
- return slow.hasNext();
- }
- });
- callables.add(callable);
- }
-
- latch.await();
-
- log.info("Starting SessionBlockVerifyIT");
-
- // let's add more for good measure.
- for (int i = 0; i < 2; i++) {
- Scanner scanner2 = c.createScanner(tableName, new Authorizations());
-
- scanner2.setRange(new Range(String.format("%08d", 0), String.format("%08d", 1000)));
-
- scanner2.setBatchSize(1);
- Iterator<Entry<Key,Value>> iter = scanner2.iterator();
- // call super's verify mechanism
- verify(iter, 0, 1000);
-
- }
-
- int sessionsFound = 0;
- // we have configured 1 tserver, so we can grab the one and only
- String tserver = Iterables.getOnlyElement(c.instanceOperations().getTabletServers());
-
- final List<ActiveScan> scans = c.instanceOperations().getActiveScans(tserver);
-
- for (ActiveScan scan : scans) {
- // only here to minimize chance of seeing meta extent scans
-
- if (tableName.equals(scan.getTable()) && scan.getSsiList().size() > 0) {
- assertEquals("Not the expected iterator", 1, scan.getSsiList().size());
- assertTrue("Not the expected iterator", scan.getSsiList().iterator().next().contains("SlowIterator"));
- sessionsFound++;
- }
-
- }
-
- /**
- * The message below indicates the problem that we experience within ACCUMULO-3509. The issue manifests as a blockage in the Scanner synchronization that
- * prevent us from making the close call against it. Since the close blocks until a read is finished, we ultimately have a block within the sweep of
- * SessionManager. As a result never reap subsequent idle sessions AND we will orphan the sessionsToCleanup in the sweep, leading to an inaccurate count
- * within sessionsFound.
- */
- assertEquals("Must have ten sessions. Failure indicates a synchronization block within the sweep mechanism", 10, sessionsFound);
- for (Future<Boolean> callable : callables) {
- callable.cancel(true);
- }
- service.shutdown();
- }
-
-}