You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2015/05/13 00:43:01 UTC
[5/5] accumulo git commit: ACCUMULO-3804 Always seal jars
ACCUMULO-3804 Always seal jars
* Move tests to packages which correspond to their jar to avoid violations.
* Make some members more visible to enable testing from ITs in other packages.
* Remove obsoleted seal-jars option from build.sh.
* Update --test option in build.sh to test everything with sealing.
* Simplify profiles, since we don't need a separate one to seal for release.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9cf9aaac
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9cf9aaac
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9cf9aaac
Branch: refs/heads/master
Commit: 9cf9aaaca5b299a9eefa29bc56f21c27bb5c1c30
Parents: d353b61
Author: Christopher Tubbs <ct...@apache.org>
Authored: Tue May 12 18:36:13 2015 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Tue May 12 18:42:00 2015 -0400
----------------------------------------------------------------------
assemble/build.sh | 9 +-
pom.xml | 32 +-
.../apache/accumulo/proxy/TestProxyClient.java | 175 --
.../server/security/SystemCredentials.java | 2 +-
.../tabletserver/LargestFirstMemoryManager.java | 2 +-
.../apache/accumulo/tserver/tablet/Tablet.java | 4 +-
.../LargestFirstMemoryManagerTest.java | 294 ----
.../tserver/LargestFirstMemoryManagerTest.java | 298 ++++
.../java/org/apache/accumulo/start/Main.java | 4 +-
.../providers/ReadOnlyHdfsFileProviderTest.java | 2 +-
.../vfs/providers/VfsClassLoaderTest.java | 2 +-
.../accumulo/start/test/AccumuloDFSBase.java | 130 ++
.../apache/accumulo/test/AccumuloDFSBase.java | 130 --
.../accumulo/fate/zookeeper/ZooLockTest.java | 379 ----
.../accumulo/proxy/ProxyDurabilityIT.java | 144 --
.../apache/accumulo/proxy/SimpleProxyBase.java | 1619 -----------------
.../apache/accumulo/proxy/TBinaryProxyIT.java | 33 -
.../apache/accumulo/proxy/TCompactProxyIT.java | 33 -
.../accumulo/proxy/TJsonProtocolProxyIT.java | 33 -
.../apache/accumulo/proxy/TTupleProxyIT.java | 33 -
.../proxy/TestProxyInstanceOperations.java | 83 -
.../accumulo/proxy/TestProxyReadWrite.java | 466 -----
.../proxy/TestProxySecurityOperations.java | 146 --
.../proxy/TestProxyTableOperations.java | 201 ---
.../server/security/SystemCredentialsIT.java | 232 ---
.../apache/accumulo/start/KeywordStartIT.java | 196 ---
.../test/fate/zookeeper/ZooLockTest.java | 381 ++++
.../accumulo/test/proxy/ProxyDurabilityIT.java | 145 ++
.../accumulo/test/proxy/SimpleProxyBase.java | 1620 ++++++++++++++++++
.../accumulo/test/proxy/TBinaryProxyIT.java | 33 +
.../accumulo/test/proxy/TCompactProxyIT.java | 33 +
.../test/proxy/TJsonProtocolProxyIT.java | 33 +
.../accumulo/test/proxy/TTupleProxyIT.java | 33 +
.../accumulo/test/proxy/TestProxyClient.java | 176 ++
.../test/proxy/TestProxyInstanceOperations.java | 84 +
.../accumulo/test/proxy/TestProxyReadWrite.java | 468 +++++
.../test/proxy/TestProxySecurityOperations.java | 147 ++
.../test/proxy/TestProxyTableOperations.java | 202 +++
.../server/security/SystemCredentialsIT.java | 233 +++
.../accumulo/test/start/KeywordStartIT.java | 197 +++
40 files changed, 4233 insertions(+), 4234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/assemble/build.sh
----------------------------------------------------------------------
diff --git a/assemble/build.sh b/assemble/build.sh
index 78ce6ca..8c339f9 100755
--- a/assemble/build.sh
+++ b/assemble/build.sh
@@ -48,17 +48,12 @@ if [[ $1 = '--create-release-candidate' ]]; then
cacheGPG
# create a release candidate from a branch
run mvn clean release:clean release:prepare release:perform
-elif [[ $1 = '--seal-jars' ]]; then
- cacheGPG
- # build a tag, but with sealed jars
- run mvn clean install \
- -P apache-release,seal-jars,thrift,assemble,docs
elif [[ $1 = '--test' ]]; then
cacheGPG
# build a tag, but with tests
run mvn clean install \
- -P apache-release,thrift,assemble,docs
+ -P apache-release,thrift,assemble,docs,accumulo-release
else
- fail "Missing one of: --create-release-candidate, --test, --seal-jars"
+ fail "Missing one of: --create-release-candidate, --test"
fi
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a6ac82c..96dd14c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -139,7 +139,6 @@
<powermock.version>1.5</powermock.version>
<!-- surefire/failsafe plugin option -->
<reuseForks>false</reuseForks>
- <sealJars>false</sealJars>
<!-- overwritten in hadoop profiles -->
<slf4j.version>1.7.5</slf4j.version>
<sourceReleaseAssemblyDescriptor>source-release-tar</sourceReleaseAssemblyDescriptor>
@@ -628,7 +627,7 @@
<configuration>
<archive>
<manifestEntries>
- <Sealed>${sealJars}</Sealed>
+ <Sealed>true</Sealed>
<Implementation-Build>${mvngit.commit.id}</Implementation-Build>
</manifestEntries>
</archive>
@@ -654,7 +653,7 @@
<goals>clean deploy</goals>
<preparationGoals>clean verify</preparationGoals>
<tagNameFormat>@{project.version}</tagNameFormat>
- <releaseProfiles>apache-release,move-source-tarball,seal-jars,skip-findbugs,!test-accumulo-maven-plugin</releaseProfiles>
+ <releaseProfiles>apache-release,accumulo-release,!test-accumulo-maven-plugin</releaseProfiles>
<useReleaseProfile>false</useReleaseProfile>
<pushChanges>false</pushChanges>
<localCheckout>true</localCheckout>
@@ -1253,7 +1252,14 @@
</reporting>
<profiles>
<profile>
- <id>move-source-tarball</id>
+ <id>accumulo-release</id>
+ <properties>
+ <!-- some properties to make the release build a bit faster -->
+ <checkstyle.skip>true</checkstyle.skip>
+ <findbugs.skip>true</findbugs.skip>
+ <skipITs>true</skipITs>
+ <skipTests>true</skipTests>
+ </properties>
<build>
<plugins>
<plugin>
@@ -1316,24 +1322,6 @@
</build>
</profile>
<profile>
- <!-- Seal jars and skip tests when the
- apache-release profile is activated. -->
- <id>seal-jars</id>
- <properties>
- <sealJars>true</sealJars>
- <skipITs>true</skipITs>
- <skipTests>true</skipTests>
- </properties>
- </profile>
- <profile>
- <!-- Skip findbugs executing when the
- apache-release profile is activated. -->
- <id>skip-findbugs</id>
- <properties>
- <findbugs.skip>true</findbugs.skip>
- </properties>
- </profile>
- <profile>
<!-- Minimal testing profile. (a.k.a. SunnyDay) -->
<id>sunny</id>
<properties>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
----------------------------------------------------------------------
diff --git a/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java b/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
deleted file mode 100644
index 99ebb38..0000000
--- a/proxy/src/main/java/org/apache/accumulo/proxy/TestProxyClient.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.proxy;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.iterators.user.RegExFilter;
-import org.apache.accumulo.proxy.thrift.AccumuloProxy;
-import org.apache.accumulo.proxy.thrift.ColumnUpdate;
-import org.apache.accumulo.proxy.thrift.Key;
-import org.apache.accumulo.proxy.thrift.ScanResult;
-import org.apache.accumulo.proxy.thrift.TimeType;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
-
-public class TestProxyClient {
-
- protected AccumuloProxy.Client proxy;
- protected TTransport transport;
-
- public TestProxyClient(String host, int port) throws TTransportException {
- this(host, port, new TCompactProtocol.Factory());
- }
-
- public TestProxyClient(String host, int port, TProtocolFactory protoFactory) throws TTransportException {
- final TSocket socket = new TSocket(host, port);
- socket.setTimeout(600000);
- transport = new TFramedTransport(socket);
- final TProtocol protocol = protoFactory.getProtocol(transport);
- proxy = new AccumuloProxy.Client(protocol);
- transport.open();
- }
-
- public AccumuloProxy.Client proxy() {
- return proxy;
- }
-
- public static void main(String[] args) throws Exception {
-
- TestProxyClient tpc = new TestProxyClient("localhost", 42424);
- String principal = "root";
- Map<String,String> props = new TreeMap<String,String>();
- props.put("password", "secret");
-
- System.out.println("Logging in");
- ByteBuffer login = tpc.proxy.login(principal, props);
-
- System.out.println("Creating user: ");
- if (!tpc.proxy().listLocalUsers(login).contains("testuser")) {
- tpc.proxy().createLocalUser(login, "testuser", ByteBuffer.wrap("testpass".getBytes(UTF_8)));
- }
- System.out.println("UserList: " + tpc.proxy().listLocalUsers(login));
-
- System.out.println("Listing: " + tpc.proxy().listTables(login));
-
- System.out.println("Deleting: ");
- String testTable = "testtableOMGOMGOMG";
-
- System.out.println("Creating: ");
-
- if (tpc.proxy().tableExists(login, testTable))
- tpc.proxy().deleteTable(login, testTable);
-
- tpc.proxy().createTable(login, testTable, true, TimeType.MILLIS);
-
- System.out.println("Listing: " + tpc.proxy().listTables(login));
-
- System.out.println("Writing: ");
- Date start = new Date();
- Date then = new Date();
- int maxInserts = 1000000;
- String format = "%1$05d";
- Map<ByteBuffer,List<ColumnUpdate>> mutations = new HashMap<ByteBuffer,List<ColumnUpdate>>();
- for (int i = 0; i < maxInserts; i++) {
- String result = String.format(format, i);
- ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(("cf" + i).getBytes(UTF_8)), ByteBuffer.wrap(("cq" + i).getBytes(UTF_8)));
- update.setValue(Util.randStringBuffer(10));
- mutations.put(ByteBuffer.wrap(result.getBytes(UTF_8)), Collections.singletonList(update));
-
- if (i % 1000 == 0) {
- tpc.proxy().updateAndFlush(login, testTable, mutations);
- mutations.clear();
- }
- }
- tpc.proxy().updateAndFlush(login, testTable, mutations);
- Date end = new Date();
- System.out.println(" End of writing: " + (end.getTime() - start.getTime()));
-
- tpc.proxy().deleteTable(login, testTable);
- tpc.proxy().createTable(login, testTable, true, TimeType.MILLIS);
-
- // Thread.sleep(1000);
-
- System.out.println("Writing async: ");
- start = new Date();
- then = new Date();
- mutations.clear();
- String writer = tpc.proxy().createWriter(login, testTable, null);
- for (int i = 0; i < maxInserts; i++) {
- String result = String.format(format, i);
- Key pkey = new Key();
- pkey.setRow(result.getBytes(UTF_8));
- ColumnUpdate update = new ColumnUpdate(ByteBuffer.wrap(("cf" + i).getBytes(UTF_8)), ByteBuffer.wrap(("cq" + i).getBytes(UTF_8)));
- update.setValue(Util.randStringBuffer(10));
- mutations.put(ByteBuffer.wrap(result.getBytes(UTF_8)), Collections.singletonList(update));
- tpc.proxy().update(writer, mutations);
- mutations.clear();
- }
-
- end = new Date();
- System.out.println(" End of writing: " + (end.getTime() - start.getTime()));
- start = end;
- System.out.println("Closing...");
- tpc.proxy().closeWriter(writer);
- end = new Date();
- System.out.println(" End of closing: " + (end.getTime() - start.getTime()));
-
- System.out.println("Reading: ");
-
- String regex = "cf1.*";
-
- IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
- RegExFilter.setRegexs(is, null, regex, null, null, false);
-
- String cookie = tpc.proxy().createScanner(login, testTable, null);
-
- int i = 0;
- start = new Date();
- then = new Date();
- boolean hasNext = true;
-
- int k = 1000;
- while (hasNext) {
- ScanResult kvList = tpc.proxy().nextK(cookie, k);
-
- Date now = new Date();
- System.out.println(i + " " + (now.getTime() - then.getTime()));
- then = now;
-
- i += kvList.getResultsSize();
- // for (TKeyValue kv:kvList.getResults()) System.out.println(new Key(kv.getKey()));
- hasNext = kvList.isMore();
- }
- end = new Date();
- System.out.println("Total entries: " + i + " total time " + (end.getTime() - start.getTime()));
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
index 0b44727..9473aca 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/SystemCredentials.java
@@ -52,7 +52,7 @@ public final class SystemCredentials extends Credentials {
private final TCredentials AS_THRIFT;
- SystemCredentials(Instance instance, String principal, AuthenticationToken token) {
+ public SystemCredentials(Instance instance, String principal, AuthenticationToken token) {
super(principal, token);
AS_THRIFT = super.toThrift(instance);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
index a39c8b6..d1bb8b5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
@@ -145,7 +145,7 @@ public class LargestFirstMemoryManager implements MemoryManager {
return mincIdleThresholds.get(tableId);
}
- boolean tableExists(Instance instance, String tableId) {
+ protected boolean tableExists(Instance instance, String tableId) {
return Tables.exists(instance, tableId);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 7eb2069..f9de6e6 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2792,8 +2792,8 @@ public class Tablet implements TabletCommitter {
}
}
- public Map<Long, List<FileRef>> getBulkIngestedFiles() {
- return new HashMap<Long, List<FileRef>>(bulkImported.asMap());
+ public Map<Long,List<FileRef>> getBulkIngestedFiles() {
+ return new HashMap<Long,List<FileRef>>(bulkImported.asMap());
}
public void cleanupBulkLoadedFiles(Set<Long> tids) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
deleted file mode 100644
index b08b980..0000000
--- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.tabletserver;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.data.impl.KeyExtent;
-import org.apache.accumulo.server.conf.NamespaceConfiguration;
-import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
-import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import com.google.common.base.Function;
-
-public class LargestFirstMemoryManagerTest {
-
- private static final long ZERO = System.currentTimeMillis();
- private static final long LATER = ZERO + 20 * 60 * 1000;
- private static final long ONE_GIG = 1024 * 1024 * 1024;
- private static final long HALF_GIG = ONE_GIG / 2;
- private static final long QGIG = ONE_GIG / 4;
- private static final long ONE_MINUTE = 60 * 1000;
-
- @Test
- public void test() throws Exception {
- LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
- ServerConfiguration config = new ServerConfiguration() {
- ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
-
- @Override
- public AccumuloConfiguration getConfiguration() {
- return DefaultConfiguration.getInstance();
- }
-
- @Override
- public TableConfiguration getTableConfiguration(String tableId) {
- return delegate.getTableConfiguration(tableId);
- }
-
- @Override
- public TableConfiguration getTableConfiguration(KeyExtent extent) {
- return delegate.getTableConfiguration(extent);
- }
-
- @Override
- public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
- return delegate.getNamespaceConfiguration(namespaceId);
- }
-
- @Override
- public Instance getInstance() {
- return delegate.getInstance();
- }
- };
- mgr.init(config);
- MemoryManagementActions result;
- // nothing to do
- result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO, 2000, 0)));
- assertEquals(0, result.tabletsToMinorCompact.size());
- // one tablet is really big
- result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"), ZERO, 2000, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
- // one tablet is idle
- mgr.currentTime = LATER;
- result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, 2000, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
- // one tablet is idle, but one is really big
- result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, ONE_GIG, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
- // lots of work to do
- mgr = new LargestFirstMemoryManagerUnderTest();
- mgr.init(config);
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
- t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0),
- t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0)));
- assertEquals(2, result.tabletsToMinorCompact.size());
- assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
- assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
- // one finished, one in progress, one filled up
- mgr = new LargestFirstMemoryManagerUnderTest();
- mgr.init(config);
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
- t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0),
- t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
- // memory is very full, lots of candidates
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
- t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0),
- t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
- assertEquals(2, result.tabletsToMinorCompact.size());
- assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
- assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
- // only have two compactors, still busy
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
- t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6),
- t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
- assertEquals(0, result.tabletsToMinorCompact.size());
- // finished one
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
- t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0),
- t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
-
- // many are running: do nothing
- mgr = new LargestFirstMemoryManagerUnderTest();
- mgr.init(config);
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
- t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG), t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG),
- t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG), t(k("m"), ZERO, 0, HALF_GIG)));
- assertEquals(0, result.tabletsToMinorCompact.size());
-
- // observe adjustment:
- mgr = new LargestFirstMemoryManagerUnderTest();
- mgr.init(config);
- // compact the largest
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
- // show that it is compacting... do nothing
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
- assertEquals(0, result.tabletsToMinorCompact.size());
- // not going to bother compacting any more
- mgr.currentTime += ONE_MINUTE;
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
- assertEquals(0, result.tabletsToMinorCompact.size());
- // now do nothing
- mgr.currentTime += ONE_MINUTE;
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, 0, 0), t(k("c"), ZERO, 0, 0)));
- assertEquals(0, result.tabletsToMinorCompact.size());
- // on no! more data, this time we compact because we've adjusted
- mgr.currentTime += ONE_MINUTE;
- result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
- }
-
- @Test
- public void testDeletedTable() throws Exception {
- final String deletedTableId = "1";
- Function<String,Boolean> existenceCheck = new Function<String,Boolean>() {
- public Boolean apply(String tableId) {
- return !deletedTableId.equals(tableId);
- }
- };
- LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
- ServerConfiguration config = new ServerConfiguration() {
- ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
-
- @Override
- public AccumuloConfiguration getConfiguration() {
- return DefaultConfiguration.getInstance();
- }
-
- @Override
- public TableConfiguration getTableConfiguration(String tableId) {
- return delegate.getTableConfiguration(tableId);
- }
-
- @Override
- public TableConfiguration getTableConfiguration(KeyExtent extent) {
- return delegate.getTableConfiguration(extent);
- }
-
- @Override
- public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
- return delegate.getNamespaceConfiguration(namespaceId);
- }
-
- @Override
- public Instance getInstance() {
- return delegate.getInstance();
- }
- };
- mgr.init(config);
- MemoryManagementActions result;
- // one tablet is really big and the other is for a nonexistent table
- KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null);
- result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0)));
- assertEquals(1, result.tabletsToMinorCompact.size());
- assertEquals(extent, result.tabletsToMinorCompact.get(0));
- }
-
- private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager {
-
- public long currentTime = ZERO;
-
- @Override
- protected long currentTimeMillis() {
- return currentTime;
- }
-
- @Override
- protected long getMinCIdleThreshold(KeyExtent extent) {
- return 15 * 60 * 1000;
- }
-
- @Override
- boolean tableExists(Instance instance, String tableId) {
- return true;
- }
- }
-
- private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest {
-
- Function<String,Boolean> existenceCheck;
-
- public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck) {
- super();
- this.existenceCheck = existenceCheck;
- }
-
- @Override
- boolean tableExists(Instance instance, String tableId) {
- return existenceCheck.apply(tableId);
- }
- }
-
- private static KeyExtent k(String endRow) {
- return new KeyExtent(new Text("1"), new Text(endRow), null);
- }
-
- private static class TestTabletState implements TabletState {
-
- private final KeyExtent extent;
- private final long lastCommit;
- private final long memSize;
- private final long compactingSize;
-
- TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize) {
- this.extent = extent;
- this.lastCommit = commit;
- this.memSize = memsize;
- this.compactingSize = compactingTableSize;
- }
-
- @Override
- public KeyExtent getExtent() {
- return extent;
- }
-
- @Override
- public long getLastCommitTime() {
- return lastCommit;
- }
-
- @Override
- public long getMemTableSize() {
- return memSize;
- }
-
- @Override
- public long getMinorCompactingMemTableSize() {
- return compactingSize;
- }
-
- }
-
- private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize) {
- return new TestTabletState(ke, lastCommit, memSize, compactingSize);
- }
-
- private static List<TabletState> tablets(TabletState... states) {
- return Arrays.asList(states);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
new file mode 100644
index 0000000..f3bd220
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.server.conf.NamespaceConfiguration;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager;
+import org.apache.accumulo.server.tabletserver.MemoryManagementActions;
+import org.apache.accumulo.server.tabletserver.TabletState;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.base.Function;
+
+public class LargestFirstMemoryManagerTest {
+
+ private static final long ZERO = System.currentTimeMillis();
+ private static final long LATER = ZERO + 20 * 60 * 1000;
+ private static final long ONE_GIG = 1024 * 1024 * 1024;
+ private static final long HALF_GIG = ONE_GIG / 2;
+ private static final long QGIG = ONE_GIG / 4;
+ private static final long ONE_MINUTE = 60 * 1000;
+
+ @Test
+ public void test() throws Exception {
+ LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest();
+ ServerConfiguration config = new ServerConfiguration() {
+ ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ return DefaultConfiguration.getInstance();
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(String tableId) {
+ return delegate.getTableConfiguration(tableId);
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(KeyExtent extent) {
+ return delegate.getTableConfiguration(extent);
+ }
+
+ @Override
+ public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
+ return delegate.getNamespaceConfiguration(namespaceId);
+ }
+
+ @Override
+ public Instance getInstance() {
+ return delegate.getInstance();
+ }
+ };
+ mgr.init(config);
+ MemoryManagementActions result;
+ // nothing to do
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO, 2000, 0)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // one tablet is really big
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"), ZERO, 2000, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+ // one tablet is idle
+ mgr.currentTime = LATER;
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, 2000, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("x"), result.tabletsToMinorCompact.get(0));
+ // one tablet is idle, but one is really big
+ result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, ONE_GIG, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("y"), result.tabletsToMinorCompact.get(0));
+ // lots of work to do
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
+ t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0),
+ t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0)));
+ assertEquals(2, result.tabletsToMinorCompact.size());
+ assertEquals(k("i"), result.tabletsToMinorCompact.get(0));
+ assertEquals(k("h"), result.tabletsToMinorCompact.get(1));
+ // one finished, one in progress, one filled up
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
+ t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0),
+ t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+ // memory is very full, lots of candidates
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
+ t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0),
+ t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+ assertEquals(2, result.tabletsToMinorCompact.size());
+ assertEquals(k("g"), result.tabletsToMinorCompact.get(0));
+ assertEquals(k("f"), result.tabletsToMinorCompact.get(1));
+ // only have two compactors, still busy
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
+ t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6),
+ t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // finished one
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0),
+ t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0),
+ t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("e"), result.tabletsToMinorCompact.get(0));
+
+ // many are running: do nothing
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0),
+ t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG), t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG),
+ t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG), t(k("m"), ZERO, 0, HALF_GIG)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+
+ // observe adjustment:
+ mgr = new LargestFirstMemoryManagerUnderTest();
+ mgr.init(config);
+ // compact the largest
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("c"), result.tabletsToMinorCompact.get(0));
+ // show that it is compacting... do nothing
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // not going to bother compacting any more
+ mgr.currentTime += ONE_MINUTE;
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // now do nothing
+ mgr.currentTime += ONE_MINUTE;
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, 0, 0), t(k("c"), ZERO, 0, 0)));
+ assertEquals(0, result.tabletsToMinorCompact.size());
+ // on no! more data, this time we compact because we've adjusted
+ mgr.currentTime += ONE_MINUTE;
+ result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(k("b"), result.tabletsToMinorCompact.get(0));
+ }
+
+ @Test
+ public void testDeletedTable() throws Exception {
+ final String deletedTableId = "1";
+ Function<String,Boolean> existenceCheck = new Function<String,Boolean>() {
+ @Override
+ public Boolean apply(String tableId) {
+ return !deletedTableId.equals(tableId);
+ }
+ };
+ LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck);
+ ServerConfiguration config = new ServerConfiguration() {
+ ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance());
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ return DefaultConfiguration.getInstance();
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(String tableId) {
+ return delegate.getTableConfiguration(tableId);
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(KeyExtent extent) {
+ return delegate.getTableConfiguration(extent);
+ }
+
+ @Override
+ public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) {
+ return delegate.getNamespaceConfiguration(namespaceId);
+ }
+
+ @Override
+ public Instance getInstance() {
+ return delegate.getInstance();
+ }
+ };
+ mgr.init(config);
+ MemoryManagementActions result;
+ // one tablet is really big and the other is for a nonexistent table
+ KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null);
+ result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0)));
+ assertEquals(1, result.tabletsToMinorCompact.size());
+ assertEquals(extent, result.tabletsToMinorCompact.get(0));
+ }
+
+ private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager {
+
+ public long currentTime = ZERO;
+
+ @Override
+ protected long currentTimeMillis() {
+ return currentTime;
+ }
+
+ @Override
+ protected long getMinCIdleThreshold(KeyExtent extent) {
+ return 15 * 60 * 1000;
+ }
+
+ @Override
+ protected boolean tableExists(Instance instance, String tableId) {
+ return true;
+ }
+ }
+
+ private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest {
+
+ Function<String,Boolean> existenceCheck;
+
+ public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck) {
+ super();
+ this.existenceCheck = existenceCheck;
+ }
+
+ @Override
+ protected boolean tableExists(Instance instance, String tableId) {
+ return existenceCheck.apply(tableId);
+ }
+ }
+
+ private static KeyExtent k(String endRow) {
+ return new KeyExtent(new Text("1"), new Text(endRow), null);
+ }
+
+ private static class TestTabletState implements TabletState {
+
+ private final KeyExtent extent;
+ private final long lastCommit;
+ private final long memSize;
+ private final long compactingSize;
+
+ TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize) {
+ this.extent = extent;
+ this.lastCommit = commit;
+ this.memSize = memsize;
+ this.compactingSize = compactingTableSize;
+ }
+
+ @Override
+ public KeyExtent getExtent() {
+ return extent;
+ }
+
+ @Override
+ public long getLastCommitTime() {
+ return lastCommit;
+ }
+
+ @Override
+ public long getMemTableSize() {
+ return memSize;
+ }
+
+ @Override
+ public long getMinorCompactingMemTableSize() {
+ return compactingSize;
+ }
+
+ }
+
+ private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize) {
+ return new TestTabletState(ke, lastCommit, memSize, compactingSize);
+ }
+
+ private static List<TabletState> tablets(TabletState... states) {
+ return Arrays.asList(states);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/start/src/main/java/org/apache/accumulo/start/Main.java
----------------------------------------------------------------------
diff --git a/start/src/main/java/org/apache/accumulo/start/Main.java b/start/src/main/java/org/apache/accumulo/start/Main.java
index abc0489..99e1d5c 100644
--- a/start/src/main/java/org/apache/accumulo/start/Main.java
+++ b/start/src/main/java/org/apache/accumulo/start/Main.java
@@ -183,14 +183,14 @@ public class Main {
System.out.println("accumulo " + kwString + " | <accumulo class> args");
}
- static Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
+ public static Map<String,KeywordExecutable> getExecutables(final ClassLoader cl) {
if (servicesMap == null) {
servicesMap = checkDuplicates(ServiceLoader.load(KeywordExecutable.class, cl));
}
return servicesMap;
}
- static Map<String,KeywordExecutable> checkDuplicates(final Iterable<? extends KeywordExecutable> services) {
+ public static Map<String,KeywordExecutable> checkDuplicates(final Iterable<? extends KeywordExecutable> services) {
TreeSet<String> blacklist = new TreeSet<>();
TreeMap<String,KeywordExecutable> results = new TreeMap<>();
for (KeywordExecutable service : services) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java
----------------------------------------------------------------------
diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java
index b5cec83..225afba 100644
--- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java
+++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/ReadOnlyHdfsFileProviderTest.java
@@ -19,7 +19,7 @@ package org.apache.accumulo.start.classloader.vfs.providers;
import java.io.IOException;
import java.util.Map;
-import org.apache.accumulo.test.AccumuloDFSBase;
+import org.apache.accumulo.start.test.AccumuloDFSBase;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.FileType;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/VfsClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/VfsClassLoaderTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/VfsClassLoaderTest.java
index 2ba4ecf..1efd7b5 100644
--- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/VfsClassLoaderTest.java
+++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/providers/VfsClassLoaderTest.java
@@ -18,7 +18,7 @@ package org.apache.accumulo.start.classloader.vfs.providers;
import java.net.URL;
-import org.apache.accumulo.test.AccumuloDFSBase;
+import org.apache.accumulo.start.test.AccumuloDFSBase;
import org.apache.commons.vfs2.FileChangeEvent;
import org.apache.commons.vfs2.FileListener;
import org.apache.commons.vfs2.FileObject;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
----------------------------------------------------------------------
diff --git a/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java b/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
new file mode 100644
index 0000000..fc5c16d
--- /dev/null
+++ b/start/src/test/java/org/apache/accumulo/start/test/AccumuloDFSBase.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
+import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
+import org.apache.commons.vfs2.CacheStrategy;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.cache.DefaultFilesCache;
+import org.apache.commons.vfs2.cache.SoftRefFilesCache;
+import org.apache.commons.vfs2.impl.DefaultFileReplicator;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class AccumuloDFSBase {
+
+ protected static Configuration conf = null;
+ protected static DefaultFileSystemManager vfs = null;
+ protected static MiniDFSCluster cluster = null;
+
+ private static URI HDFS_URI;
+
+ protected static URI getHdfsUri() {
+ return HDFS_URI;
+ }
+
+ @BeforeClass
+ public static void miniDfsClusterSetup() {
+ System.setProperty("java.io.tmpdir", System.getProperty("user.dir") + "/target");
+ // System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
+ // Logger.getRootLogger().setLevel(Level.ERROR);
+
+ // Put the MiniDFSCluster directory in the target directory
+ System.setProperty("test.build.data", "target/build/test/data");
+
+ // Setup HDFS
+ conf = new Configuration();
+ conf.set("hadoop.security.token.service.use_ip", "true");
+
+ conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M blocksize
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitClusterUp();
+ // We can't assume that the hostname of "localhost" will still be "localhost" after
+ // starting up the NameNode. We may get mapped into a FQDN via settings in /etc/hosts.
+ HDFS_URI = cluster.getFileSystem().getUri();
+ } catch (IOException e) {
+ throw new RuntimeException("Error setting up mini cluster", e);
+ }
+
+ // Set up the VFS
+ vfs = new DefaultFileSystemManager();
+ try {
+ vfs.setFilesCache(new DefaultFilesCache());
+ vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
+ vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
+ vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
+ vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
+ vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
+ vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
+ vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
+ vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
+ vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
+ vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
+ vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
+ vfs.addProvider("hdfs", new HdfsFileProvider());
+ vfs.addExtensionMap("jar", "jar");
+ vfs.addExtensionMap("zip", "zip");
+ vfs.addExtensionMap("gz", "gz");
+ vfs.addExtensionMap("tar", "tar");
+ vfs.addExtensionMap("tbz2", "tar");
+ vfs.addExtensionMap("tgz", "tar");
+ vfs.addExtensionMap("bz2", "bz2");
+ vfs.addMimeTypeMap("application/x-tar", "tar");
+ vfs.addMimeTypeMap("application/x-gzip", "gz");
+ vfs.addMimeTypeMap("application/zip", "zip");
+ vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
+ vfs.setFilesCache(new SoftRefFilesCache());
+ vfs.setReplicator(new DefaultFileReplicator(new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-"
+ + System.getProperty("user.name", "nouser"))));
+ vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
+ vfs.init();
+ } catch (FileSystemException e) {
+ throw new RuntimeException("Error setting up VFS", e);
+ }
+
+ }
+
+ @AfterClass
+ public static void tearDownMiniDfsCluster() {
+ if (null != cluster) {
+ cluster.shutdown();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java
----------------------------------------------------------------------
diff --git a/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java b/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java
deleted file mode 100644
index 3e94291..0000000
--- a/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
-import org.apache.commons.vfs2.CacheStrategy;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.cache.DefaultFilesCache;
-import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
-import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-public class AccumuloDFSBase {
-
- protected static Configuration conf = null;
- protected static DefaultFileSystemManager vfs = null;
- protected static MiniDFSCluster cluster = null;
-
- private static URI HDFS_URI;
-
- protected static URI getHdfsUri() {
- return HDFS_URI;
- }
-
- @BeforeClass
- public static void miniDfsClusterSetup() {
- System.setProperty("java.io.tmpdir", System.getProperty("user.dir") + "/target");
- // System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
- // Logger.getRootLogger().setLevel(Level.ERROR);
-
- // Put the MiniDFSCluster directory in the target directory
- System.setProperty("test.build.data", "target/build/test/data");
-
- // Setup HDFS
- conf = new Configuration();
- conf.set("hadoop.security.token.service.use_ip", "true");
-
- conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M blocksize
-
- try {
- cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitClusterUp();
- // We can't assume that the hostname of "localhost" will still be "localhost" after
- // starting up the NameNode. We may get mapped into a FQDN via settings in /etc/hosts.
- HDFS_URI = cluster.getFileSystem().getUri();
- } catch (IOException e) {
- throw new RuntimeException("Error setting up mini cluster", e);
- }
-
- // Set up the VFS
- vfs = new DefaultFileSystemManager();
- try {
- vfs.setFilesCache(new DefaultFilesCache());
- vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
- vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
- vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
- vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
- vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
- vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
- vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
- vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
- vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
- vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
- vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
- vfs.addProvider("hdfs", new HdfsFileProvider());
- vfs.addExtensionMap("jar", "jar");
- vfs.addExtensionMap("zip", "zip");
- vfs.addExtensionMap("gz", "gz");
- vfs.addExtensionMap("tar", "tar");
- vfs.addExtensionMap("tbz2", "tar");
- vfs.addExtensionMap("tgz", "tar");
- vfs.addExtensionMap("bz2", "bz2");
- vfs.addMimeTypeMap("application/x-tar", "tar");
- vfs.addMimeTypeMap("application/x-gzip", "gz");
- vfs.addMimeTypeMap("application/zip", "zip");
- vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
- vfs.setFilesCache(new SoftRefFilesCache());
- vfs.setReplicator(new DefaultFileReplicator(new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-"
- + System.getProperty("user.name", "nouser"))));
- vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
- vfs.init();
- } catch (FileSystemException e) {
- throw new RuntimeException("Error setting up VFS", e);
- }
-
- }
-
- @AfterClass
- public static void tearDownMiniDfsCluster() {
- if (null != cluster) {
- cluster.shutdown();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java b/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
deleted file mode 100644
index 4c21a1f..0000000
--- a/test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.fate.zookeeper;
-
-import java.io.File;
-import java.lang.reflect.Field;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.accumulo.fate.zookeeper.ZooLock.AsyncLockWatcher;
-import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
-import org.apache.accumulo.minicluster.MiniAccumuloCluster;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.Watcher.Event.KeeperState;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooKeeper;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- *
- */
-public class ZooLockTest {
-
- private static final TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
-
- private static MiniAccumuloCluster accumulo;
-
- static class ConnectedWatcher implements Watcher {
- volatile boolean connected = false;
-
- @Override
- public synchronized void process(WatchedEvent event) {
- if (event.getState() == KeeperState.SyncConnected) { // For ZK >3.4.... || event.getState() == KeeperState.ConnectedReadOnly) {
- connected = true;
- } else {
- connected = false;
- }
- }
-
- public synchronized boolean isConnected() {
- return connected;
- }
- }
-
- static class TestALW implements AsyncLockWatcher {
-
- LockLossReason reason = null;
- boolean locked = false;
- Exception exception = null;
- int changes = 0;
-
- @Override
- public synchronized void lostLock(LockLossReason reason) {
- this.reason = reason;
- changes++;
- this.notifyAll();
- }
-
- @Override
- public synchronized void acquiredLock() {
- this.locked = true;
- changes++;
- this.notifyAll();
- }
-
- @Override
- public synchronized void failedToAcquireLock(Exception e) {
- this.exception = e;
- changes++;
- this.notifyAll();
- }
-
- public synchronized void waitForChanges(int numExpected) throws InterruptedException {
- while (changes < numExpected) {
- this.wait();
- }
- }
-
- @Override
- public synchronized void unableToMonitorLockNode(Throwable e) {
- changes++;
- this.notifyAll();
- }
- }
-
- @BeforeClass
- public static void setupMiniCluster() throws Exception {
-
- folder.create();
-
- accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret");
-
- accumulo.start();
-
- }
-
- private static final AtomicInteger pdCount = new AtomicInteger(0);
-
- @Test(timeout = 10000)
- public void testDeleteParent() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- Assert.assertFalse(zl.isLocked());
-
- ZooReaderWriter zk = ZooReaderWriter.getInstance(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes());
-
- // intentionally created parent after lock
- zk.mkdirs(parent);
-
- zk.delete(parent, -1);
-
- zk.mkdirs(parent);
-
- TestALW lw = new TestALW();
-
- zl.lockAsync(lw, "test1".getBytes());
-
- lw.waitForChanges(1);
-
- Assert.assertTrue(lw.locked);
- Assert.assertTrue(zl.isLocked());
- Assert.assertNull(lw.exception);
- Assert.assertNull(lw.reason);
-
- zl.unlock();
- }
-
- @Test(timeout = 10000)
- public void testNoParent() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- Assert.assertFalse(zl.isLocked());
-
- TestALW lw = new TestALW();
-
- zl.lockAsync(lw, "test1".getBytes());
-
- lw.waitForChanges(1);
-
- Assert.assertFalse(lw.locked);
- Assert.assertFalse(zl.isLocked());
- Assert.assertNotNull(lw.exception);
- Assert.assertNull(lw.reason);
- }
-
- @Test(timeout = 10000)
- public void testDeleteLock() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
-
- ZooReaderWriter zk = ZooReaderWriter.getInstance(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes());
- zk.mkdirs(parent);
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- Assert.assertFalse(zl.isLocked());
-
- TestALW lw = new TestALW();
-
- zl.lockAsync(lw, "test1".getBytes());
-
- lw.waitForChanges(1);
-
- Assert.assertTrue(lw.locked);
- Assert.assertTrue(zl.isLocked());
- Assert.assertNull(lw.exception);
- Assert.assertNull(lw.reason);
-
- zk.delete(zl.getLockPath(), -1);
-
- lw.waitForChanges(2);
-
- Assert.assertEquals(LockLossReason.LOCK_DELETED, lw.reason);
- Assert.assertNull(lw.exception);
-
- }
-
- @Test(timeout = 10000)
- public void testDeleteWaiting() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
-
- ZooReaderWriter zk = ZooReaderWriter.getInstance(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes());
- zk.mkdirs(parent);
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- Assert.assertFalse(zl.isLocked());
-
- TestALW lw = new TestALW();
-
- zl.lockAsync(lw, "test1".getBytes());
-
- lw.waitForChanges(1);
-
- Assert.assertTrue(lw.locked);
- Assert.assertTrue(zl.isLocked());
- Assert.assertNull(lw.exception);
- Assert.assertNull(lw.reason);
-
- ZooLock zl2 = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- TestALW lw2 = new TestALW();
-
- zl2.lockAsync(lw2, "test2".getBytes());
-
- Assert.assertFalse(lw2.locked);
- Assert.assertFalse(zl2.isLocked());
-
- ZooLock zl3 = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- TestALW lw3 = new TestALW();
-
- zl3.lockAsync(lw3, "test3".getBytes());
-
- List<String> children = zk.getChildren(parent);
- Collections.sort(children);
-
- zk.delete(parent + "/" + children.get(1), -1);
-
- lw2.waitForChanges(1);
-
- Assert.assertFalse(lw2.locked);
- Assert.assertNotNull(lw2.exception);
- Assert.assertNull(lw2.reason);
-
- zk.delete(parent + "/" + children.get(0), -1);
-
- lw.waitForChanges(2);
-
- Assert.assertEquals(LockLossReason.LOCK_DELETED, lw.reason);
- Assert.assertNull(lw.exception);
-
- lw3.waitForChanges(1);
-
- Assert.assertTrue(lw3.locked);
- Assert.assertTrue(zl3.isLocked());
- Assert.assertNull(lw3.exception);
- Assert.assertNull(lw3.reason);
-
- zl3.unlock();
-
- }
-
- @Test(timeout = 10000)
- public void testUnexpectedEvent() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
-
- ConnectedWatcher watcher = new ConnectedWatcher();
- ZooKeeper zk = new ZooKeeper(accumulo.getZooKeepers(), 30000, watcher);
- zk.addAuthInfo("digest", "secret".getBytes());
-
- while (!watcher.isConnected()) {
- Thread.sleep(200);
- }
-
- zk.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 30000, "digest", "secret".getBytes(), parent);
-
- Assert.assertFalse(zl.isLocked());
-
- // would not expect data to be set on this node, but it should not cause problems.....
- zk.setData(parent, "foo".getBytes(), -1);
-
- TestALW lw = new TestALW();
-
- zl.lockAsync(lw, "test1".getBytes());
-
- lw.waitForChanges(1);
-
- Assert.assertTrue(lw.locked);
- Assert.assertTrue(zl.isLocked());
- Assert.assertNull(lw.exception);
- Assert.assertNull(lw.reason);
-
- // would not expect data to be set on this node either
- zk.setData(zl.getLockPath(), "bar".getBytes(), -1);
-
- zk.delete(zl.getLockPath(), -1);
-
- lw.waitForChanges(2);
-
- Assert.assertEquals(LockLossReason.LOCK_DELETED, lw.reason);
- Assert.assertNull(lw.exception);
-
- }
-
- @Test(timeout = 10000)
- public void testTryLock() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 1000, "digest", "secret".getBytes(), parent);
-
- ConnectedWatcher watcher = new ConnectedWatcher();
- ZooKeeper zk = new ZooKeeper(accumulo.getZooKeepers(), 1000, watcher);
- zk.addAuthInfo("digest", "secret".getBytes());
-
- while (!watcher.isConnected()) {
- Thread.sleep(200);
- }
-
- for (int i = 0; i < 10; i++) {
- zk.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- zk.delete(parent, -1);
- }
-
- zk.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-
- TestALW lw = new TestALW();
-
- boolean ret = zl.tryLock(lw, "test1".getBytes());
-
- Assert.assertTrue(ret);
-
- // make sure still watching parent even though a lot of events occurred for the parent
- synchronized (zl) {
- Field field = zl.getClass().getDeclaredField("watchingParent");
- field.setAccessible(true);
- Assert.assertTrue((Boolean) field.get(zl));
- }
-
- zl.unlock();
- }
-
- @Test(timeout = 10000)
- public void testChangeData() throws Exception {
- String parent = "/zltest-" + this.hashCode() + "-l" + pdCount.incrementAndGet();
- ConnectedWatcher watcher = new ConnectedWatcher();
- ZooKeeper zk = new ZooKeeper(accumulo.getZooKeepers(), 1000, watcher);
- zk.addAuthInfo("digest", "secret".getBytes());
-
- while (!watcher.isConnected()) {
- Thread.sleep(200);
- }
-
- zk.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-
- ZooLock zl = new ZooLock(accumulo.getZooKeepers(), 1000, "digest", "secret".getBytes(), parent);
-
- TestALW lw = new TestALW();
-
- zl.lockAsync(lw, "test1".getBytes());
- Assert.assertEquals("test1", new String(zk.getData(zl.getLockPath(), null, null)));
-
- zl.replaceLockData("test2".getBytes());
- Assert.assertEquals("test2", new String(zk.getData(zl.getLockPath(), null, null)));
- }
-
- @AfterClass
- public static void tearDownMiniCluster() throws Exception {
- accumulo.stop();
- folder.delete();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf9aaac/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
deleted file mode 100644
index 53653da..0000000
--- a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.proxy;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
-import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
-import org.apache.accumulo.proxy.thrift.Column;
-import org.apache.accumulo.proxy.thrift.ColumnUpdate;
-import org.apache.accumulo.proxy.thrift.Condition;
-import org.apache.accumulo.proxy.thrift.ConditionalStatus;
-import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
-import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
-import org.apache.accumulo.proxy.thrift.Durability;
-import org.apache.accumulo.proxy.thrift.TimeType;
-import org.apache.accumulo.proxy.thrift.WriterOptions;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.thrift.protocol.TJSONProtocol;
-import org.apache.thrift.server.TServer;
-import org.junit.Test;
-
-import com.google.common.collect.Iterators;
-import com.google.common.net.HostAndPort;
-
-public class ProxyDurabilityIT extends ConfigurableMacIT {
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 60;
- }
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
- cfg.setNumTservers(1);
- }
-
- private static ByteBuffer bytes(String value) {
- return ByteBuffer.wrap(value.getBytes());
- }
-
- @Test
- public void testDurability() throws Exception {
- Connector c = getConnector();
- Properties props = new Properties();
- // Avoid issues with locally installed client configuration files with custom properties
- File emptyFile = Files.createTempFile(null, null).toFile();
- emptyFile.deleteOnExit();
- props.put("instance", c.getInstance().getInstanceName());
- props.put("zookeepers", c.getInstance().getZooKeepers());
- props.put("tokenClass", PasswordToken.class.getName());
- props.put("clientConfigurationFile", emptyFile.toString());
-
- TJSONProtocol.Factory protocol = new TJSONProtocol.Factory();
-
- int proxyPort = PortUtils.getRandomFreePort();
- final TServer proxyServer = Proxy.createProxyServer(HostAndPort.fromParts("localhost", proxyPort), protocol, props).server;
- while (!proxyServer.isServing())
- UtilWaitThread.sleep(100);
- Client client = new TestProxyClient("localhost", proxyPort, protocol).proxy();
- Map<String,String> properties = new TreeMap<String,String>();
- properties.put("password", ROOT_PASSWORD);
- ByteBuffer login = client.login("root", properties);
-
- String tableName = getUniqueNames(1)[0];
- client.createTable(login, tableName, true, TimeType.MILLIS);
- assertTrue(c.tableOperations().exists(tableName));
-
- WriterOptions options = new WriterOptions();
- options.setDurability(Durability.NONE);
- String writer = client.createWriter(login, tableName, options);
- Map<ByteBuffer,List<ColumnUpdate>> cells = new TreeMap<ByteBuffer,List<ColumnUpdate>>();
- ColumnUpdate column = new ColumnUpdate(bytes("cf"), bytes("cq"));
- column.setValue("value".getBytes());
- cells.put(bytes("row"), Collections.singletonList(column));
- client.update(writer, cells);
- client.closeWriter(writer);
- assertEquals(1, count(tableName));
- restartTServer();
- assertEquals(0, count(tableName));
-
- ConditionalWriterOptions cfg = new ConditionalWriterOptions();
- cfg.setDurability(Durability.SYNC);
- String cwriter = client.createConditionalWriter(login, tableName, cfg);
- ConditionalUpdates updates = new ConditionalUpdates();
- updates.addToConditions(new Condition(new Column(bytes("cf"), bytes("cq"), bytes(""))));
- updates.addToUpdates(column);
- Map<ByteBuffer,ConditionalStatus> status = client.updateRowsConditionally(cwriter, Collections.singletonMap(bytes("row"), updates));
- assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes("row")));
- assertEquals(1, count(tableName));
- restartTServer();
- assertEquals(1, count(tableName));
-
- proxyServer.stop();
- }
-
- private void restartTServer() throws Exception {
- for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
- cluster.killProcess(ServerType.TABLET_SERVER, proc);
- }
- cluster.start();
- }
-
- private int count(String tableName) throws Exception {
- return Iterators.size((getConnector().createScanner(tableName, Authorizations.EMPTY)).iterator());
- }
-
-}