You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2015/06/04 20:52:48 UTC
[07/43] accumulo git commit: ACCUMULO-3871 move ITs into distro jar,
stop building test jar
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java b/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
deleted file mode 100644
index 07d197d..0000000
--- a/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import static org.junit.Assert.assertTrue;
-
-import java.net.Socket;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Range;
-import com.google.common.net.HostAndPort;
-
-// ACCUMULO-2757 - make sure we don't make too many more watchers
-public class WatchTheWatchCountIT extends ConfigurableMacBase {
- private static final Logger log = LoggerFactory.getLogger(WatchTheWatchCountIT.class);
-
- public int defaultOverrideSeconds() {
- return 60;
- }
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setNumTservers(3);
- }
-
- @Test
- public void test() throws Exception {
- Connector c = getConnector();
- String[] tableNames = getUniqueNames(3);
- for (String tableName : tableNames) {
- c.tableOperations().create(tableName);
- }
- c.tableOperations().list();
- String zooKeepers = c.getInstance().getZooKeepers();
- final Range<Long> expectedWatcherRange = Range.open(475l, 700l);
- long total = 0;
- final HostAndPort hostAndPort = HostAndPort.fromString(zooKeepers);
- for (int i = 0; i < 5; i++) {
- Socket socket = new Socket(hostAndPort.getHostText(), hostAndPort.getPort());
- try {
- socket.getOutputStream().write("wchs\n".getBytes(), 0, 5);
- byte[] buffer = new byte[1024];
- int n = socket.getInputStream().read(buffer);
- String response = new String(buffer, 0, n);
- total = Long.parseLong(response.split(":")[1].trim());
- log.info("Total: {}", total);
- if (expectedWatcherRange.contains(total)) {
- break;
- }
- log.debug("Expected number of watchers to be contained in {}, but actually was {}. Sleeping and retrying", expectedWatcherRange, total);
- Thread.sleep(5000);
- } finally {
- socket.close();
- }
- }
-
- assertTrue("Expected number of watchers to be contained in " + expectedWatcherRange + ", but actually was " + total, expectedWatcherRange.contains(total));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java b/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
deleted file mode 100644
index d877969..0000000
--- a/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.TestIngest;
-import org.apache.accumulo.test.VerifyIngest;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.junit.Test;
-
-public class WriteAheadLogIT extends AccumuloClusterHarness {
-
- @Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
- cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
- cfg.setProperty(Property.GC_CYCLE_START, "1");
- cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "1s");
- cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "4s");
- hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- }
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 10 * 60;
- }
-
- @Test
- public void test() throws Exception {
- Connector c = getConnector();
- String tableName = getUniqueNames(1)[0];
- c.tableOperations().create(tableName);
- c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "750K");
- TestIngest.Opts opts = new TestIngest.Opts();
- VerifyIngest.Opts vopts = new VerifyIngest.Opts();
- opts.setTableName(tableName);
-
- ClientConfiguration clientConfig = cluster.getClientConfig();
- if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
- opts.updateKerberosCredentials(clientConfig);
- vopts.updateKerberosCredentials(clientConfig);
- } else {
- opts.setPrincipal(getAdminPrincipal());
- vopts.setPrincipal(getAdminPrincipal());
- }
-
- TestIngest.ingest(c, opts, new BatchWriterOpts());
- vopts.setTableName(tableName);
- VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
- getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
- getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
- VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java b/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
deleted file mode 100644
index 45b671c..0000000
--- a/test/src/test/java/org/apache/accumulo/test/functional/WriteLotsIT.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.ClientConfiguration;
-import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
-import org.apache.accumulo.test.TestIngest;
-import org.apache.accumulo.test.VerifyIngest;
-import org.junit.Test;
-
-public class WriteLotsIT extends AccumuloClusterHarness {
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 90;
- }
-
- @Test
- public void writeLots() throws Exception {
- final Connector c = getConnector();
- final String tableName = getUniqueNames(1)[0];
- c.tableOperations().create(tableName);
- final AtomicReference<Exception> ref = new AtomicReference<Exception>();
- List<Thread> threads = new ArrayList<Thread>();
- final ClientConfiguration clientConfig = getCluster().getClientConfig();
- for (int i = 0; i < 10; i++) {
- final int index = i;
- Thread t = new Thread() {
- @Override
- public void run() {
- try {
- TestIngest.Opts opts = new TestIngest.Opts();
- opts.startRow = index * 10000;
- opts.rows = 10000;
- opts.setTableName(tableName);
- if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
- opts.updateKerberosCredentials(clientConfig);
- } else {
- opts.setPrincipal(getAdminPrincipal());
- }
- TestIngest.ingest(c, opts, new BatchWriterOpts());
- } catch (Exception ex) {
- ref.set(ex);
- }
- }
- };
- t.start();
- threads.add(t);
- }
- for (Thread thread : threads) {
- thread.join();
- }
- if (ref.get() != null) {
- throw ref.get();
- }
- VerifyIngest.Opts vopts = new VerifyIngest.Opts();
- vopts.rows = 10000 * 10;
- vopts.setTableName(tableName);
- if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
- vopts.updateKerberosCredentials(clientConfig);
- } else {
- vopts.setPrincipal(getAdminPrincipal());
- }
- VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/functional/ZooCacheIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ZooCacheIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ZooCacheIT.java
deleted file mode 100644
index a531ee0..0000000
--- a/test/src/test/java/org/apache/accumulo/test/functional/ZooCacheIT.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ZooCacheIT extends ConfigurableMacBase {
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 2 * 60;
- }
-
- private static String pathName = "/zcTest-42";
- private static File testDir;
-
- @BeforeClass
- public static void createTestDirectory() {
- testDir = new File(createTestDir(ZooCacheIT.class.getName()), pathName);
- FileUtils.deleteQuietly(testDir);
- assertTrue(testDir.mkdir());
- }
-
- @Test
- public void test() throws Exception {
- assertEquals(0, exec(CacheTestClean.class, pathName, testDir.getAbsolutePath()).waitFor());
- final AtomicReference<Exception> ref = new AtomicReference<Exception>();
- List<Thread> threads = new ArrayList<Thread>();
- for (int i = 0; i < 3; i++) {
- Thread reader = new Thread() {
- @Override
- public void run() {
- try {
- CacheTestReader.main(new String[] {pathName, testDir.getAbsolutePath(), getConnector().getInstance().getZooKeepers()});
- } catch (Exception ex) {
- ref.set(ex);
- }
- }
- };
- reader.start();
- threads.add(reader);
- }
- assertEquals(0, exec(CacheTestWriter.class, pathName, testDir.getAbsolutePath(), "3", "50").waitFor());
- for (Thread t : threads) {
- t.join();
- if (ref.get() != null)
- throw ref.get();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
deleted file mode 100644
index 19f90fe..0000000
--- a/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.functional;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-public class ZookeeperRestartIT extends ConfigurableMacBase {
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
- cfg.setSiteConfig(siteConfig);
- }
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 2 * 60;
- }
-
- @Test
- public void test() throws Exception {
- Connector c = getConnector();
- c.tableOperations().create("test_ingest");
- BatchWriter bw = c.createBatchWriter("test_ingest", null);
- Mutation m = new Mutation("row");
- m.put("cf", "cq", "value");
- bw.addMutation(m);
- bw.close();
-
- // kill zookeeper
- for (ProcessReference proc : cluster.getProcesses().get(ServerType.ZOOKEEPER))
- cluster.killProcess(ServerType.ZOOKEEPER, proc);
-
- // give the servers time to react
- UtilWaitThread.sleep(1000);
-
- // start zookeeper back up
- cluster.start();
-
- // use the tservers
- Scanner s = c.createScanner("test_ingest", Authorizations.EMPTY);
- Iterator<Entry<Key,Value>> i = s.iterator();
- assertTrue(i.hasNext());
- assertEquals("row", i.next().getKey().getRow().toString());
- assertFalse(i.hasNext());
- // use the master
- c.tableOperations().delete("test_ingest");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/test/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
deleted file mode 100644
index a0d355e..0000000
--- a/test/src/test/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
-import org.apache.accumulo.test.continuous.ContinuousIngest;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-public class RollWALPerformanceIT extends ConfigurableMacBase {
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1");
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "10M");
- cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100");
- cfg.setProperty(Property.GC_FILE_ARCHIVE, "false");
- cfg.setProperty(Property.GC_CYCLE_START, "1s");
- cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
- cfg.useMiniDFS(true);
- }
-
- private long ingest() throws Exception {
- final Connector c = getConnector();
- final String tableName = getUniqueNames(1)[0];
-
- log.info("Creating the table");
- c.tableOperations().create(tableName);
-
- log.info("Splitting the table");
- final long SPLIT_COUNT = 100;
- final long distance = Long.MAX_VALUE / SPLIT_COUNT;
- final SortedSet<Text> splits = new TreeSet<Text>();
- for (int i = 1; i < SPLIT_COUNT; i++) {
- splits.add(new Text(String.format("%016x", i * distance)));
- }
- c.tableOperations().addSplits(tableName, splits);
-
- log.info("Waiting for balance");
- c.instanceOperations().waitForBalance();
-
- final Instance inst = c.getInstance();
-
- log.info("Starting ingest");
- final long start = System.currentTimeMillis();
- final String args[] = {"-i", inst.getInstanceName(), "-z", inst.getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "--batchThreads", "2", "--table",
- tableName, "--num", Long.toString(1000 * 1000), // 1M 100 byte entries
- };
-
- ContinuousIngest.main(args);
- final long result = System.currentTimeMillis() - start;
- log.debug(String.format("Finished in %,d ms", result));
- log.debug("Dropping table");
- c.tableOperations().delete(tableName);
- return result;
- }
-
- private long getAverage() throws Exception {
- final int REPEAT = 3;
- long totalTime = 0;
- for (int i = 0; i < REPEAT; i++) {
- totalTime += ingest();
- }
- return totalTime / REPEAT;
- }
-
- private void testWalPerformanceOnce() throws Exception {
- // get time with a small WAL, which will cause many WAL roll-overs
- long avg1 = getAverage();
- // use a bigger WAL max size to eliminate WAL roll-overs
- Connector c = getConnector();
- c.instanceOperations().setProperty(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1G");
- c.tableOperations().flush(MetadataTable.NAME, null, null, true);
- c.tableOperations().flush(RootTable.NAME, null, null, true);
- for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
- getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
- }
- getCluster().start();
- long avg2 = getAverage();
- log.info(String.format("Average run time with small WAL %,d with large WAL %,d", avg1, avg2));
- assertTrue(avg1 > avg2);
- double percent = (100. * avg1) / avg2;
- log.info(String.format("Percent of large log: %.2f%%", percent));
- assertTrue(percent < 125.);
- }
-
- @Test(timeout = 20 * 60 * 1000)
- public void testWalPerformance() throws Exception {
- testWalPerformanceOnce();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java b/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
deleted file mode 100644
index 236522a..0000000
--- a/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.performance.metadata;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.FileOperations;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.RFile;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-// ACCUMULO-3327
-public class FastBulkImportIT extends ConfigurableMacBase {
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 60;
- }
-
- @Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setNumTservers(3);
- cfg.setProperty(Property.TSERV_BULK_ASSIGNMENT_THREADS, "5");
- cfg.setProperty(Property.TSERV_BULK_PROCESS_THREADS, "5");
- cfg.setProperty(Property.TABLE_MAJC_RATIO, "9999");
- cfg.setProperty(Property.TABLE_FILE_MAX, "9999");
- }
-
- @Test
- public void test() throws Exception {
- log.info("Creating table");
- final String tableName = getUniqueNames(1)[0];
- final Connector c = getConnector();
- c.tableOperations().create(tableName);
- log.info("Adding splits");
- SortedSet<Text> splits = new TreeSet<>();
- for (int i = 1; i < 0xfff; i += 7) {
- splits.add(new Text(Integer.toHexString(i)));
- }
- c.tableOperations().addSplits(tableName, splits);
-
- log.info("Creating lots of bulk import files");
- FileSystem fs = getCluster().getFileSystem();
- Path basePath = getCluster().getTemporaryPath();
- CachedConfiguration.setInstance(fs.getConf());
-
- Path base = new Path(basePath, "testBulkFail_" + tableName);
- fs.delete(base, true);
- fs.mkdirs(base);
- Path bulkFailures = new Path(base, "failures");
- Path files = new Path(base, "files");
- fs.mkdirs(bulkFailures);
- fs.mkdirs(files);
- for (int i = 0; i < 100; i++) {
- FileSKVWriter writer = FileOperations.getInstance().openWriter(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf(),
- AccumuloConfiguration.getDefaultConfiguration());
- writer.startDefaultLocalityGroup();
- for (int j = 0x100; j < 0xfff; j += 3) {
- writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
- }
- writer.close();
- }
- log.info("Waiting for balance");
- c.instanceOperations().waitForBalance();
-
- log.info("Bulk importing files");
- long now = System.currentTimeMillis();
- c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), true);
- double diffSeconds = (System.currentTimeMillis() - now) / 1000.;
- log.info(String.format("Import took %.2f seconds", diffSeconds));
- assertTrue(diffSeconds < 30);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/01ae5b85/test/src/test/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java b/test/src/test/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
deleted file mode 100644
index 745326e..0000000
--- a/test/src/test/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.proxy;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
-import org.apache.accumulo.proxy.Proxy;
-import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
-import org.apache.accumulo.proxy.thrift.Column;
-import org.apache.accumulo.proxy.thrift.ColumnUpdate;
-import org.apache.accumulo.proxy.thrift.Condition;
-import org.apache.accumulo.proxy.thrift.ConditionalStatus;
-import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
-import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
-import org.apache.accumulo.proxy.thrift.Durability;
-import org.apache.accumulo.proxy.thrift.TimeType;
-import org.apache.accumulo.proxy.thrift.WriterOptions;
-import org.apache.accumulo.server.util.PortUtils;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.thrift.protocol.TJSONProtocol;
-import org.apache.thrift.server.TServer;
-import org.junit.Test;
-
-import com.google.common.collect.Iterators;
-import com.google.common.net.HostAndPort;
-
-public class ProxyDurabilityIT extends ConfigurableMacBase {
-
- @Override
- protected int defaultTimeoutSeconds() {
- return 60;
- }
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
- cfg.setNumTservers(1);
- }
-
- private static ByteBuffer bytes(String value) {
- return ByteBuffer.wrap(value.getBytes());
- }
-
- @Test
- public void testDurability() throws Exception {
- Connector c = getConnector();
- Properties props = new Properties();
- // Avoid issues with locally installed client configuration files with custom properties
- File emptyFile = Files.createTempFile(null, null).toFile();
- emptyFile.deleteOnExit();
- props.put("instance", c.getInstance().getInstanceName());
- props.put("zookeepers", c.getInstance().getZooKeepers());
- props.put("tokenClass", PasswordToken.class.getName());
- props.put("clientConfigurationFile", emptyFile.toString());
-
- TJSONProtocol.Factory protocol = new TJSONProtocol.Factory();
-
- int proxyPort = PortUtils.getRandomFreePort();
- final TServer proxyServer = Proxy.createProxyServer(HostAndPort.fromParts("localhost", proxyPort), protocol, props).server;
- while (!proxyServer.isServing())
- UtilWaitThread.sleep(100);
- Client client = new TestProxyClient("localhost", proxyPort, protocol).proxy();
- Map<String,String> properties = new TreeMap<String,String>();
- properties.put("password", ROOT_PASSWORD);
- ByteBuffer login = client.login("root", properties);
-
- String tableName = getUniqueNames(1)[0];
- client.createTable(login, tableName, true, TimeType.MILLIS);
- assertTrue(c.tableOperations().exists(tableName));
-
- WriterOptions options = new WriterOptions();
- options.setDurability(Durability.NONE);
- String writer = client.createWriter(login, tableName, options);
- Map<ByteBuffer,List<ColumnUpdate>> cells = new TreeMap<ByteBuffer,List<ColumnUpdate>>();
- ColumnUpdate column = new ColumnUpdate(bytes("cf"), bytes("cq"));
- column.setValue("value".getBytes());
- cells.put(bytes("row"), Collections.singletonList(column));
- client.update(writer, cells);
- client.closeWriter(writer);
- assertEquals(1, count(tableName));
- restartTServer();
- assertEquals(0, count(tableName));
-
- ConditionalWriterOptions cfg = new ConditionalWriterOptions();
- cfg.setDurability(Durability.SYNC);
- String cwriter = client.createConditionalWriter(login, tableName, cfg);
- ConditionalUpdates updates = new ConditionalUpdates();
- updates.addToConditions(new Condition(new Column(bytes("cf"), bytes("cq"), bytes(""))));
- updates.addToUpdates(column);
- Map<ByteBuffer,ConditionalStatus> status = client.updateRowsConditionally(cwriter, Collections.singletonMap(bytes("row"), updates));
- assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes("row")));
- assertEquals(1, count(tableName));
- restartTServer();
- assertEquals(1, count(tableName));
-
- proxyServer.stop();
- }
-
- private void restartTServer() throws Exception {
- for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
- cluster.killProcess(ServerType.TABLET_SERVER, proc);
- }
- cluster.start();
- }
-
- private int count(String tableName) throws Exception {
- return Iterators.size((getConnector().createScanner(tableName, Authorizations.EMPTY)).iterator());
- }
-
-}