You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2019/02/01 01:27:25 UTC
[accumulo] branch master updated (db7c429 -> 07933f0)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from db7c429 Merge branch '2.0.0-alpha-2-rc1-next'
new f9ca165 Fix javadoc html syntax
new 07933f0 Fix #924 Ensure scanners remain open for test
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../accumulo/core/client/AccumuloClient.java | 8 +--
.../apache/accumulo/test/functional/ScanIdIT.java | 82 ++++++++++++++--------
2 files changed, 56 insertions(+), 34 deletions(-)
[accumulo] 01/02: Fix javadoc html syntax
Posted by ct...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f9ca165510a77b2e27dbc8266a6190cff4aa5ba7
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Thu Jan 31 19:36:24 2019 -0500
Fix javadoc html syntax
---
.../main/java/org/apache/accumulo/core/client/AccumuloClient.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/client/AccumuloClient.java b/core/src/main/java/org/apache/accumulo/core/client/AccumuloClient.java
index ee6a627..d3cde65 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/AccumuloClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/AccumuloClient.java
@@ -55,7 +55,7 @@ import org.apache.accumulo.core.security.Authorizations;
* AccumuloClient and pass it around.
*
* @since 2.0.0
- * @see <a href=https://accumulo.apache.org/docs/2.x/getting-started/clients>Accumulo Client
+ * @see <a href="https://accumulo.apache.org/docs/2.x/getting-started/clients">Accumulo Client
* Documentation</a>
*/
public interface AccumuloClient extends AutoCloseable {
@@ -359,7 +359,7 @@ public interface AccumuloClient extends AutoCloseable {
* @param propertiesFilePath
* Path to properties file
* @return this builder
- * @see <a href=https://accumulo.apache.org/docs/2.x/configuration/client-properties>Client
+ * @see <a href="https://accumulo.apache.org/docs/2.x/configuration/client-properties">Client
* properties documentation</a>
*/
FromOptions<T> from(String propertiesFilePath);
@@ -371,7 +371,7 @@ public interface AccumuloClient extends AutoCloseable {
* @param propertiesFile
* Path to properties file
* @return this builder
- * @see <a href=https://accumulo.apache.org/docs/2.x/configuration/client-properties>Client
+ * @see <a href="https://accumulo.apache.org/docs/2.x/configuration/client-properties">Client
* properties documentation</a>
*/
FromOptions<T> from(Path propertiesFile);
@@ -383,7 +383,7 @@ public interface AccumuloClient extends AutoCloseable {
* @param properties
* Properties object
* @return this builder
- * @see <a href=https://accumulo.apache.org/docs/2.x/configuration/client-properties>Client
+ * @see <a href="https://accumulo.apache.org/docs/2.x/configuration/client-properties">Client
* properties documentation</a>
*/
FromOptions<T> from(Properties properties);
[accumulo] 02/02: Fix #924 Ensure scanners remain open for test
Posted by ct...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 07933f08e3c7054c64d33378fd0a28c723c95a1b
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Thu Jan 31 20:14:20 2019 -0500
Fix #924 Ensure scanners remain open for test
Ensure scanners remain open, and not cleaned up by closing them, for the
duration of the test, so that the test can gather the unique scan IDs.
---
.../apache/accumulo/test/functional/ScanIdIT.java | 82 ++++++++++++++--------
1 file changed, 52 insertions(+), 30 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ScanIdIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ScanIdIT.java
index 6785a69..ebb6d39 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ScanIdIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ScanIdIT.java
@@ -18,11 +18,13 @@ package org.apache.accumulo.test.functional;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.security.SecureRandom;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@@ -48,6 +50,7 @@ import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.ActiveScan;
+import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
@@ -124,8 +127,10 @@ public class ScanIdIT extends AccumuloClusterHarness {
CountDownLatch latch = new CountDownLatch(NUM_SCANNERS);
+ List<ScannerThread> scanThreadsToClose = new ArrayList<>(NUM_SCANNERS);
for (int scannerIndex = 0; scannerIndex < NUM_SCANNERS; scannerIndex++) {
ScannerThread st = new ScannerThread(client, scannerIndex, tableName, latch);
+ scanThreadsToClose.add(st);
pool.submit(st);
}
@@ -147,44 +152,64 @@ public class ScanIdIT extends AccumuloClusterHarness {
}
- // all scanner have reported at least 1 result, so check for unique scan ids.
- Set<Long> scanIds = new HashSet<>();
+ Set<Long> scanIds = getScanIds(client);
+ assertTrue("Expected at least " + NUM_SCANNERS + " scanIds, but saw " + scanIds.size(),
+ scanIds.size() >= NUM_SCANNERS);
- List<String> tservers = client.instanceOperations().getTabletServers();
+ scanThreadsToClose.forEach(st -> {
+ if (st.scanner != null) {
+ st.scanner.close();
+ }
+ });
- log.debug("tablet servers {}", tservers);
+ while (!(scanIds = getScanIds(client)).isEmpty()) {
+ log.debug("Waiting for active scans to stop...");
+ Thread.sleep(200);
+ }
+ assertEquals("Expected no scanIds after closing scanners", 0, scanIds.size());
- for (String tserver : tservers) {
+ }
+ }
- List<ActiveScan> activeScans = null;
- for (int i = 0; i < 10; i++) {
- try {
- activeScans = client.instanceOperations().getActiveScans(tserver);
- break;
- } catch (AccumuloException e) {
- if (e.getCause() instanceof TableNotFoundException) {
- log.debug("Got TableNotFoundException, will retry");
- Thread.sleep(200);
- continue;
- }
- throw e;
- }
- }
+ private Set<Long> getScanIds(AccumuloClient client)
+ throws AccumuloSecurityException, InterruptedException, AccumuloException {
+ // all scanner have reported at least 1 result, so check for unique scan ids.
+ Set<Long> scanIds = new HashSet<>();
- assertNotNull("Repeatedly got exception trying to active scans", activeScans);
+ List<String> tservers = client.instanceOperations().getTabletServers();
- log.debug("TServer {} has {} active scans", tserver, activeScans.size());
+ log.debug("tablet servers {}", tservers);
- for (ActiveScan scan : activeScans) {
- log.debug("Tserver {} scan id {}", tserver, scan.getScanid());
- scanIds.add(scan.getScanid());
+ for (String tserver : tservers) {
+
+ List<ActiveScan> activeScans = null;
+ for (int i = 0; i < 10; i++) {
+ try {
+ activeScans = client.instanceOperations().getActiveScans(tserver);
+ break;
+ } catch (AccumuloException e) {
+ if (e.getCause() instanceof TableNotFoundException) {
+ log.debug("Got TableNotFoundException, will retry");
+ Thread.sleep(200);
+ continue;
+ }
+ throw e;
}
}
- assertTrue("Expected at least " + NUM_SCANNERS + " scanIds, but saw " + scanIds.size(),
- scanIds.size() >= NUM_SCANNERS);
+ assertNotNull("Repeatedly got exception trying to active scans", activeScans);
+ activeScans
+ .removeIf(scan -> scan.getTable().startsWith(Namespace.ACCUMULO + Namespace.SEPARATOR));
+ log.debug("TServer {} has {} active non-metadata scans", tserver, activeScans.size());
+
+ for (ActiveScan scan : activeScans) {
+ log.debug("Tserver {} scan id {} ({})", tserver, scan.getScanid(), scan.getTable());
+ scanIds.add(scan.getScanid());
+ }
}
+
+ return scanIds;
}
/**
@@ -244,7 +269,6 @@ public class ScanIdIT extends AccumuloClusterHarness {
// exit when success condition is met.
if (!testInProgress.get()) {
scanner.clearScanIterators();
- scanner.close();
return;
}
@@ -274,9 +298,7 @@ public class ScanIdIT extends AccumuloClusterHarness {
} catch (TableNotFoundException e) {
throw new IllegalStateException("Initialization failure. Could not create scanner", e);
} finally {
- if (scanner != null) {
- scanner.close();
- }
+ // don't close scanner here, because it will clean up the scan ids we're checking for
}
}
}