You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/07/02 22:57:23 UTC
svn commit: r1499110 [1/3] - in /accumulo/trunk:
core/src/main/java/org/apache/accumulo/core/conf/
server/src/main/java/org/apache/accumulo/server/util/
test/src/main/java/org/apache/accumulo/test/
test/src/main/java/org/apache/accumulo/test/functional...
Author: ecn
Date: Tue Jul 2 20:57:21 2013
New Revision: 1499110
URL: http://svn.apache.org/r1499110
Log:
ACCUMULO-1537 converted many more functional tests to integration tests
Added:
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FateStarvationIT.java (contents, props changed)
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LargeRowTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LogicalTimeIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LogicalTimeTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MaxOpenIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeMetaIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/PermissionsIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/PermissionsTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/RowDeleteIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/RowDeleteTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/ScanRangeIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ScanRangeTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/ScanSessionTimeOutIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ScanSessionTimeOutTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/ServerSideErrorIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ServerSideErrorTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/ShutdownIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/SimpleBalancerFairnessIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/SparseColumnFamilyIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/SparseColumnFamilyTest.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/SplitIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/StartIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/TableIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/TabletIT.java (with props)
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/TimeoutIT.java
- copied, changed from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/TimeoutTest.java
Removed:
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/GCLotsOfCandidatesTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LargeRowTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LogicalTimeTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/MergeTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/PermissionsTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/RowDeleteTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ScanRangeTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ScanSessionTimeOutTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/ServerSideErrorTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/SparseColumnFamilyTest.java
accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/TimeoutTest.java
accumulo/trunk/test/system/auto/simple/examples.py
accumulo/trunk/test/system/auto/simple/fateStartvation.py
accumulo/trunk/test/system/auto/simple/gc.py
accumulo/trunk/test/system/auto/simple/largeRow.py
accumulo/trunk/test/system/auto/simple/logicalTime.py
accumulo/trunk/test/system/auto/simple/mapreduce.py
accumulo/trunk/test/system/auto/simple/maxOpen.py
accumulo/trunk/test/system/auto/simple/merge.py
accumulo/trunk/test/system/auto/simple/mergeMetadata.py
accumulo/trunk/test/system/auto/simple/range.py
accumulo/trunk/test/system/auto/simple/rowDelete.py
accumulo/trunk/test/system/auto/simple/scanIter.py
accumulo/trunk/test/system/auto/simple/scanSessionTimeout.py
accumulo/trunk/test/system/auto/simple/security.py
accumulo/trunk/test/system/auto/simple/serverSideError.py
accumulo/trunk/test/system/auto/simple/shutdown.py
accumulo/trunk/test/system/auto/simple/simpleBalancer.py
accumulo/trunk/test/system/auto/simple/sparseColumnFamily.py
accumulo/trunk/test/system/auto/simple/split.py
accumulo/trunk/test/system/auto/simple/start.py
accumulo/trunk/test/system/auto/simple/table.py
accumulo/trunk/test/system/auto/simple/tablets.py
accumulo/trunk/test/system/auto/simple/timeout.py
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java?rev=1499110&r1=1499109&r2=1499110&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java Tue Jul 2 20:57:21 2013
@@ -49,7 +49,7 @@ public class SiteConfiguration extends A
xmlConfig = new Configuration(false);
if (SiteConfiguration.class.getClassLoader().getResource(configFile) == null)
- log.warn(configFile + " not found on classpath");
+ log.warn(configFile + " not found on classpath", new Throwable());
else
xmlConfig.addResource(configFile);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java?rev=1499110&r1=1499109&r2=1499110&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java Tue Jul 2 20:57:21 2013
@@ -195,7 +195,7 @@ public class CheckForMetadataProblems {
checkMetadataTableEntries(opts, fs);
opts.stopTracing();
if (sawProblems)
- System.exit(-1);
+ throw new RuntimeException();
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1499110&r1=1499109&r2=1499110&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Tue Jul 2 20:57:21 2013
@@ -64,6 +64,7 @@ import org.apache.accumulo.core.util.Str
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.ServerConstants;
@@ -73,7 +74,6 @@ import org.apache.accumulo.server.fs.Vol
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.security.SecurityConstants;
-import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java?rev=1499110&r1=1499109&r2=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DeleteIT.java Tue Jul 2 20:57:21 2013
@@ -32,17 +32,22 @@ public class DeleteIT extends MacTest {
@Test(timeout=60*1000)
public void test() throws Exception {
Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ deleteTest(c);
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ }
+
+ public static void deleteTest(Connector c) throws Exception {
VerifyIngest.Opts vopts = new VerifyIngest.Opts();
TestIngest.Opts opts = new TestIngest.Opts();
- vopts.rows = opts.rows = 10000;
+ vopts.rows = opts.rows = 1000;
vopts.cols = opts.cols = 1;
vopts.random = opts.random = 56;
- opts.createTable = true;
TestIngest.ingest(c, opts, new BatchWriterOpts());
assertEquals(0, cluster.exec(TestRandomDeletes.class, "-p", MacTest.PASSWORD, "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers()).waitFor());
TestIngest.ingest(c, opts, new BatchWriterOpts());
VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
- assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
}
+
}
Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java?rev=1499110&r1=1499109&r2=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/DynamicThreadPoolsIT.java Tue Jul 2 20:57:21 2013
@@ -16,8 +16,7 @@
*/
package org.apache.accumulo.test.functional;
-import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.util.Collections;
Copied: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FateStarvationIT.java (from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationTest.java)
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FateStarvationIT.java?p2=accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FateStarvationIT.java&p1=accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationTest.java&r1=1498909&r2=1499110&rev=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/FateStarvationTest.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FateStarvationIT.java Tue Jul 2 20:57:21 2013
@@ -17,42 +17,36 @@
package org.apache.accumulo.test.functional;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Random;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.test.TestIngest;
import org.apache.hadoop.io.Text;
+import org.junit.Test;
/**
* See ACCUMULO-779
*/
-public class FateStarvationTest extends FunctionalTest {
+public class FateStarvationIT extends MacTest {
- @Override
- public void cleanup() throws Exception {}
-
- @Override
- public Map<String,String> getInitialConfig() {
- return Collections.emptyMap();
- }
-
- @Override
- public List<TableSetup> getTablesToCreate() {
- return Collections.emptyList();
- }
-
- @Override
+ @Test(timeout=60*1000)
public void run() throws Exception {
- getConnector().tableOperations().create("test_ingest");
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
- getConnector().tableOperations().addSplits("test_ingest", TestIngest.getSplitPoints(0, 100000, 50));
+ c.tableOperations().addSplits("test_ingest", TestIngest.getSplitPoints(0, 100000, 50));
- TestIngest.main(new String[] {"-random", "89", "-timestamp", "7", "-size", "" + 50, "100000", "0", "1"});
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.random = 89;
+ opts.timestamp = 7;
+ opts.dataSize = 50;
+ opts.rows = 100000;
+ opts.cols = 1;
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
- getConnector().tableOperations().flush("test_ingest", null, null, true);
+ c.tableOperations().flush("test_ingest", null, null, true);
List<Text> splits = new ArrayList<Text>(TestIngest.getSplitPoints(0, 100000, 67));
Random rand = new Random();
@@ -61,17 +55,10 @@ public class FateStarvationTest extends
int idx1 = rand.nextInt(splits.size() - 1);
int idx2 = rand.nextInt(splits.size() - (idx1 + 1)) + idx1 + 1;
- getConnector().tableOperations().compact("test_ingest", splits.get(idx1), splits.get(idx2), false, false);
+ c.tableOperations().compact("test_ingest", splits.get(idx1), splits.get(idx2), false, false);
}
- getConnector().tableOperations().offline("test_ingest");
+ c.tableOperations().offline("test_ingest");
}
- public static void main(String[] args) throws Exception {
- ArrayList<String> argsList = new ArrayList<String>();
- argsList.addAll(Arrays.asList(args));
- argsList.addAll(Arrays.asList(FateStarvationTest.class.getName(), "run"));
- FunctionalTest.main(argsList.toArray(new String[0]));
- }
-
}
Propchange: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FateStarvationIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java?rev=1499110&r1=1499109&r2=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java Tue Jul 2 20:57:21 2013
@@ -34,6 +34,7 @@ import org.apache.accumulo.core.cli.Batc
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
@@ -149,6 +150,15 @@ public class FunctionalTestUtils {
return readAll(new FileInputStream(c.getConfig().getLogDir() + "/" + klass.getSimpleName() + "_" + p.hashCode() + ".out"));
}
+ static Mutation nm(String row, String cf, String cq, Value value) {
+ Mutation m = new Mutation(new Text(row));
+ m.put(new Text(cf), new Text(cq), value);
+ return m;
+ }
+
+ static Mutation nm(String row, String cf, String cq, String value) {
+ return nm(row, cf, cq, new Value(value.getBytes()));
+ }
}
Added: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java?rev=1499110&view=auto
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java (added)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java Tue Jul 2 20:57:21 2013
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.gc.SimpleGarbageCollector;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class GarbageCollectorIT extends MacTest {
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String, String> settings = new HashMap<String, String>();
+ settings.put(Property.GC_CYCLE_START.getKey(), "1");
+ settings.put(Property.GC_CYCLE_DELAY.getKey(), "1");
+ settings.put(Property.TSERV_MAXMEM.getKey(), "5K");
+ settings.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
+ cfg.setSiteConfig(settings);
+ }
+
+ @Test(timeout=60*1000)
+ public void gcTest() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
+ TestIngest.Opts opts = new TestIngest.Opts();
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.rows = opts.rows = 10000;
+ vopts.cols = opts.cols = 1;
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ c.tableOperations().compact("test_ingest", null, null, true, true);
+ int before = countFiles();
+ while (true) {
+ UtilWaitThread.sleep(1000);
+ int more = countFiles();
+ if (more <= before)
+ break;
+ before = more;
+ }
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ UtilWaitThread.sleep(5*1000);
+ int after = countFiles();
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ assertTrue(after < before);
+ gc.destroy();
+ }
+
+ @Test(timeout=60*1000)
+ public void gcLotsOfCandidatesIT() throws Exception {
+ log.info("Filling !METADATA table with bogus delete flags");
+ Connector c = getConnector();
+ addEntries(c, new BatchWriterOpts());
+ cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE);
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ UtilWaitThread.sleep(10*1000);
+ String output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc);
+ gc.destroy();
+ assertTrue(output.contains("delete candidates has exceeded"));
+ }
+
+ private int countFiles() throws Exception {
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ int result = 0;
+ Path path = new Path(cluster.getConfig().getDir()+"/accumulo/tables/1/*/*.rf");
+ for (@SuppressWarnings("unused") FileStatus entry : fs.globStatus(path)) {
+ result++;
+ }
+ return result;
+ }
+
+ public static void addEntries(Connector conn, BatchWriterOpts bwOpts) throws Exception {
+ conn.securityOperations().grantTablePermission(conn.whoami(), MetadataTable.NAME, TablePermission.WRITE);
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, bwOpts.getBatchWriterConfig());
+
+ for (int i = 0; i < 100000; ++i) {
+ final Text emptyText = new Text("");
+ Text row = new Text(String.format("%s%s%020d%s", MetadataTable.DELETED_RANGE.getStartKey().getRow().toString(), "/", i,
+ "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj"));
+ Mutation delFlag = new Mutation(row);
+ delFlag.put(emptyText, emptyText, new Value(new byte[] {}));
+ bw.addMutation(delFlag);
+ }
+ bw.close();
+ }
+}
Propchange: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java (from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LargeRowTest.java)
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java?p2=accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java&p1=accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LargeRowTest.java&r1=1498909&r2=1499110&rev=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LargeRowTest.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LargeRowIT.java Tue Jul 2 20:57:21 2013
@@ -16,15 +16,14 @@
*/
package org.apache.accumulo.test.functional;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.Collections;
import java.util.Map.Entry;
import java.util.Random;
import java.util.TreeSet;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
@@ -33,12 +32,19 @@ import org.apache.accumulo.core.data.Ran
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.test.TestIngest;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
+import org.junit.Test;
-public class LargeRowTest extends FunctionalTest {
+public class LargeRowIT extends MacTest {
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ cfg.setSiteConfig(Collections.singletonMap(Property.TSERV_MAJC_DELAY.getKey(), "10ms"));
+ }
+
private static final int SEED = 42;
private static final String REG_TABLE_NAME = "lr";
private static final String PRE_SPLIT_TABLE_NAME = "lrps";
@@ -47,67 +53,44 @@ public class LargeRowTest extends Functi
private static final int NUM_PRE_SPLITS = 9;
private static final int SPLIT_THRESH = ROW_SIZE * NUM_ROWS / NUM_PRE_SPLITS;
- @Override
- public void cleanup() {}
-
- @Override
- public Map<String,String> getInitialConfig() {
- return parseConfig(Property.TSERV_MAJC_DELAY + "=10ms");
- }
-
- @Override
- public List<TableSetup> getTablesToCreate() {
-
+ @Test(timeout=60*1000)
+ public void run() throws Exception {
Random r = new Random();
byte rowData[] = new byte[ROW_SIZE];
r.setSeed(SEED + 1);
-
TreeSet<Text> splitPoints = new TreeSet<Text>();
-
for (int i = 0; i < NUM_PRE_SPLITS; i++) {
r.nextBytes(rowData);
TestIngest.toPrintableChars(rowData);
splitPoints.add(new Text(rowData));
}
-
- ArrayList<TableSetup> tables = new ArrayList<TableSetup>();
-
- tables.add(new TableSetup(REG_TABLE_NAME));
- tables.add(new TableSetup(PRE_SPLIT_TABLE_NAME, splitPoints));
-
- return tables;
- // return Collections.singletonList(new TableSetup(TABLE_NAME));
- }
-
- @Override
- public void run() throws Exception {
-
- // Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
- // logger.setLevel(Level.TRACE);
-
- test1();
- test2();
+ Connector c = getConnector();
+ c.tableOperations().create(REG_TABLE_NAME);
+ c.tableOperations().create(PRE_SPLIT_TABLE_NAME);
+ c.tableOperations().addSplits(PRE_SPLIT_TABLE_NAME, splitPoints);
+ test1(c);
+ test2(c);
}
- private void test1() throws Exception {
+ private void test1(Connector c) throws Exception {
- basicTest(REG_TABLE_NAME, 0);
+ basicTest(c, REG_TABLE_NAME, 0);
- getConnector().tableOperations().setProperty(REG_TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "" + SPLIT_THRESH);
+ c.tableOperations().setProperty(REG_TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "" + SPLIT_THRESH);
UtilWaitThread.sleep(12000);
- Logger.getLogger(LargeRowTest.class).warn("checking splits");
- checkSplits(REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
+ Logger.getLogger(LargeRowIT.class).warn("checking splits");
+ FunctionalTestUtils.checkSplits(c, REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
- verify(REG_TABLE_NAME);
+ verify(c, REG_TABLE_NAME);
}
- private void test2() throws Exception {
- basicTest(PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
+ private void test2(Connector c) throws Exception {
+ basicTest(c, PRE_SPLIT_TABLE_NAME, NUM_PRE_SPLITS);
}
- private void basicTest(String table, int expectedSplits) throws Exception {
- BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ private void basicTest(Connector c, String table, int expectedSplits) throws Exception {
+ BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig());
Random r = new Random();
byte rowData[] = new byte[ROW_SIZE];
@@ -126,34 +109,34 @@ public class LargeRowTest extends Functi
bw.close();
- checkSplits(table, expectedSplits, expectedSplits);
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
- verify(table);
+ verify(c, table);
- checkSplits(table, expectedSplits, expectedSplits);
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
- getConnector().tableOperations().flush(table, null, null, false);
+ c.tableOperations().flush(table, null, null, false);
// verify while table flush is running
- verify(table);
+ verify(c, table);
// give split time to complete
- getConnector().tableOperations().flush(table, null, null, true);
+ c.tableOperations().flush(table, null, null, true);
- checkSplits(table, expectedSplits, expectedSplits);
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
- verify(table);
+ verify(c, table);
- checkSplits(table, expectedSplits, expectedSplits);
+ FunctionalTestUtils.checkSplits(c, table, expectedSplits, expectedSplits);
}
- private void verify(String table) throws Exception {
+ private void verify(Connector c, String table) throws Exception {
Random r = new Random();
byte rowData[] = new byte[ROW_SIZE];
r.setSeed(SEED);
- Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+ Scanner scanner = c.createScanner(table, Authorizations.EMPTY);
for (int i = 0; i < NUM_ROWS; i++) {
Copied: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LogicalTimeIT.java (from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LogicalTimeTest.java)
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LogicalTimeIT.java?p2=accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LogicalTimeIT.java&p1=accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LogicalTimeTest.java&r1=1498909&r2=1499110&rev=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/LogicalTimeTest.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/LogicalTimeIT.java Tue Jul 2 20:57:21 2013
@@ -16,9 +16,6 @@
*/
package org.apache.accumulo.test.functional;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
import java.util.TreeSet;
import org.apache.accumulo.core.client.BatchWriter;
@@ -30,49 +27,42 @@ import org.apache.accumulo.core.data.Mut
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class LogicalTimeIT extends MacTest {
-public class LogicalTimeTest extends FunctionalTest {
-
- @Override
- public Map<String,String> getInitialConfig() {
- return Collections.emptyMap();
- }
- @Override
- public List<TableSetup> getTablesToCreate() {
- return Collections.emptyList();
- }
- @Override
+ @Test(timeout=120*1000)
public void run() throws Exception {
int tc = 0;
-
- runMergeTest("foo" + tc++, new String[] {"m"}, new String[] {"a"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"m"}, new String[] {"z"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"m"}, new String[] {"a", "z"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"m"}, new String[] {"a", "c", "z"}, null, null, "b", 3l);
- runMergeTest("foo" + tc++, new String[] {"m"}, new String[] {"a", "y", "z"}, null, null, "b", 3l);
-
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"h"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"s"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s"}, null, null, "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "c", "h", "s"}, null, null, "b", 3l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s", "i"}, null, null, "b", 3l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"t", "a", "h", "s"}, null, null, "b", 3l);
-
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a"}, null, "h", "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"h"}, null, "h", "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"s"}, null, "h", "b", 1l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s"}, null, "h", "b", 2l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "c", "h", "s"}, null, "h", "b", 3l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s", "i"}, null, "h", "b", 3l);
- runMergeTest("foo" + tc++, new String[] {"g", "r"}, new String[] {"t", "a", "h", "s"}, null, "h", "b", 2l);
+ Connector c = getConnector();
+ runMergeTest(c, "foo" + tc++, new String[] {"m"}, new String[] {"a"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"m"}, new String[] {"z"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"m"}, new String[] {"a", "z"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"m"}, new String[] {"a", "c", "z"}, null, null, "b", 3l);
+ runMergeTest(c, "foo" + tc++, new String[] {"m"}, new String[] {"a", "y", "z"}, null, null, "b", 3l);
+
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"h"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"s"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s"}, null, null, "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "c", "h", "s"}, null, null, "b", 3l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s", "i"}, null, null, "b", 3l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"t", "a", "h", "s"}, null, null, "b", 3l);
+
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a"}, null, "h", "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"h"}, null, "h", "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"s"}, null, "h", "b", 1l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s"}, null, "h", "b", 2l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "c", "h", "s"}, null, "h", "b", 3l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"a", "h", "s", "i"}, null, "h", "b", 3l);
+ runMergeTest(c, "foo" + tc++, new String[] {"g", "r"}, new String[] {"t", "a", "h", "s"}, null, "h", "b", 2l);
}
- private void runMergeTest(String table, String[] splits, String[] inserts, String start, String end, String last, long expected) throws Exception {
- Connector conn = super.getConnector();
+ private void runMergeTest(Connector conn, String table, String[] splits, String[] inserts, String start, String end, String last, long expected) throws Exception {
+ log.info("table " + table);
conn.tableOperations().create(table, true, TimeType.LOGICAL);
TreeSet<Text> splitSet = new TreeSet<Text>();
for (String split : splits) {
@@ -106,7 +96,4 @@ public class LogicalTimeTest extends Fun
throw new RuntimeException("unexpected time " + time + " " + expected);
}
- @Override
- public void cleanup() throws Exception {}
-
}
Added: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java?rev=1499110&view=auto
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java (added)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java Tue Jul 2 20:57:21 2013
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+
+import java.security.MessageDigest;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.examples.simple.mapreduce.RowHash;
+import org.apache.hadoop.io.Text;
+import org.codehaus.plexus.util.Base64;
+import org.junit.Test;
+
+public class MapReduceIT extends MacTest {
+
+ static final String tablename = "mapredf";
+ static final String input_cf = "cf-HASHTYPE";
+ static final String input_cq = "cq-NOTHASHED";
+ static final String input_cfcq = input_cf + ":" + input_cq;
+ static final String output_cq = "cq-MD4BASE64";
+ static final String output_cfcq = input_cf + ":" + output_cq;
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create(tablename);
+ BatchWriter bw = c.createBatchWriter(tablename, new BatchWriterConfig());
+ for (int i = 0; i < 10; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put(input_cf, input_cq, "row" + i);
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ Process hash = cluster.exec(RowHash.class,
+ "-i", cluster.getInstanceName(),
+ "-z", cluster.getZooKeepers(),
+ "-u", "root",
+ "-p", MacTest.PASSWORD,
+ "-t", tablename,
+ "--column", input_cfcq);
+ assertEquals(0, hash.waitFor());
+
+ Scanner s = c.createScanner(tablename, Authorizations.EMPTY);
+ s.fetchColumn(new Text(input_cf), new Text(output_cq));
+ int i = 0;
+ for (Entry<Key,Value> entry : s) {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ byte[] check = Base64.encodeBase64(md.digest(("row" + i).getBytes()));
+ assertEquals(entry.getValue().toString(), new String(check));
+ i++;
+ }
+
+ }
+
+
+}
Propchange: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MapReduceIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MaxOpenIT.java (from r1498909, accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenTest.java)
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MaxOpenIT.java?p2=accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MaxOpenIT.java&p1=accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenTest.java&r1=1498909&r2=1499110&rev=1499110&view=diff
==============================================================================
--- accumulo/trunk/test/src/main/java/org/apache/accumulo/test/functional/MaxOpenTest.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MaxOpenIT.java Tue Jul 2 20:57:21 2013
@@ -17,56 +17,61 @@
package org.apache.accumulo.test.functional;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.VerifyIngest;
+import org.junit.Test;
/**
- * A functional test that excercises hitting the max open file limit on a tablet server. This test assumes there are one or two tablet servers.
- *
- *
+ * A functional test that exercises hitting the max open file limit on a tablet server. This test assumes there are one or two tablet servers.
*/
-public class MaxOpenTest extends FunctionalTest {
-
- private static final int NUM_TABLETS = 16;
- private static final int NUM_TO_INGEST = 10000;
+public class MaxOpenIT extends MacTest {
@Override
- public void cleanup() {}
-
- @Override
- public Map<String,String> getInitialConfig() {
- return parseConfig(Property.TSERV_SCAN_MAX_OPENFILES + "=4", Property.TSERV_MAJC_MAXCONCURRENT + "=1", Property.TSERV_MAJC_THREAD_MAXOPEN + "=2");
- }
-
- @Override
- public List<TableSetup> getTablesToCreate() {
- Map<String,String> config = parseConfig(Property.TABLE_MAJC_RATIO + "=10");
- TableSetup ts = new TableSetup("test_ingest", config, TestIngest.getSplitPoints(0, NUM_TO_INGEST, NUM_TABLETS));
- return Collections.singletonList(ts);
+ public void configure(MiniAccumuloConfig cfg) {
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "4");
+ conf.put(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "1");
+ conf.put(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), "2");
+ cfg.setSiteConfig(conf);
}
+
+ private static final int NUM_TABLETS = 16;
+ private static final int NUM_TO_INGEST = 10000;
- @Override
+ @Test
public void run() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().setProperty("test_ingest", Property.TABLE_MAJC_RATIO.getKey(), "10");
+ c.tableOperations().addSplits("test_ingest", TestIngest.getSplitPoints(0, NUM_TO_INGEST, NUM_TABLETS));
// the following loop should create three tablets in each map file
for (int i = 0; i < 3; i++) {
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.timestamp = i;
+ opts.dataSize = 50;
+ opts.rows = NUM_TO_INGEST;
+ opts.cols = 1;
+ opts.random = i;
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
- TestIngest.main(new String[] {"-random", "" + i, "-timestamp", "" + i, "-size", "" + 50, "" + NUM_TO_INGEST, "0", "1"});
-
- getConnector().tableOperations().flush("test_ingest", null, null, true);
- checkRFiles("test_ingest", NUM_TABLETS, NUM_TABLETS, i + 1, i + 1);
+ c.tableOperations().flush("test_ingest", null, null, true);
+ FunctionalTestUtils.checkRFiles(c, "test_ingest", NUM_TABLETS, NUM_TABLETS, i + 1, i + 1);
}
List<Range> ranges = new ArrayList<Range>(NUM_TO_INGEST);
@@ -75,18 +80,18 @@ public class MaxOpenTest extends Functio
ranges.add(new Range(TestIngest.generateRow(i, 0)));
}
- long time1 = batchScan(ranges, 1);
+ long time1 = batchScan(c, ranges, 1);
// run it again, now that stuff is cached on the client and sever
- time1 = batchScan(ranges, 1);
- long time2 = batchScan(ranges, NUM_TABLETS);
+ time1 = batchScan(c, ranges, 1);
+ long time2 = batchScan(c, ranges, NUM_TABLETS);
System.out.printf("Single thread scan time %6.2f %n", time1 / 1000.0);
System.out.printf("Multiple thread scan time %6.2f %n", time2 / 1000.0);
}
- private long batchScan(List<Range> ranges, int threads) throws Exception {
- BatchScanner bs = getConnector().createBatchScanner("test_ingest", TestIngest.AUTHS, threads);
+ private long batchScan(Connector c, List<Range> ranges, int threads) throws Exception {
+ BatchScanner bs = c.createBatchScanner("test_ingest", TestIngest.AUTHS, threads);
bs.setRanges(ranges);
Added: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeIT.java?rev=1499110&view=auto
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeIT.java (added)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeIT.java Tue Jul 2 20:57:21 2013
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.Merge;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class MergeIT extends MacTest {
+
+ SortedSet<Text> splits(String [] points) {
+ SortedSet<Text> result = new TreeSet<Text>();
+ for (String point : points)
+ result.add(new Text(point));
+ return result;
+ }
+
+ @Test(timeout=30*1000)
+ public void merge() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("test");
+ c.tableOperations().addSplits("test", splits("a b c d e f g h i j k".split(" ")));
+ BatchWriter bw = c.createBatchWriter("test", new BatchWriterConfig());
+ for (String row : "a b c d e f g h i j k".split(" ")) {
+ Mutation m = new Mutation(row);
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ }
+ bw.close();
+ c.tableOperations().flush("test", null, null, true);
+ c.tableOperations().merge("test", new Text("c1"), new Text("f1"));
+ assertEquals(8, c.tableOperations().listSplits("test").size());
+ }
+
+ @Test(timeout=30*1000)
+ public void mergeSize() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("merge");
+ c.tableOperations().addSplits("merge", splits("a b c d e f g h i j k l m n o p q r s t u v w x y z".split(" ")));
+ BatchWriter bw = c.createBatchWriter("merge", new BatchWriterConfig());
+ for (String row : "c e f y".split(" ")) {
+ Mutation m = new Mutation(row);
+ m.put("cf", "cq", "mersydotesanddozeydotesanlittolamsiedives");
+ bw.addMutation(m);
+ }
+ bw.close();
+ c.tableOperations().flush("merge", null, null, true);
+ Merge merge = new Merge();
+ merge.mergomatic(c, "merge", null, null, 100, false);
+ assertArrayEquals("b c d e f x y".split(" "), toStrings(c.tableOperations().listSplits("merge")));
+ merge.mergomatic(c, "merge", null, null, 100, true);
+ assertArrayEquals("c e f y".split(" "), toStrings(c.tableOperations().listSplits("merge")));
+ }
+
+ private String[] toStrings(Collection<Text> listSplits) {
+ String[] result = new String[listSplits.size()];
+ int i = 0;
+ for (Text t : listSplits) {
+ result[i++] = t.toString();
+ }
+ return result;
+ }
+
+ private String[] ns(String... strings) {
+ return strings;
+ }
+
+ @Test(timeout=120*1000)
+ public void mergeTest() throws Exception {
+ int tc = 0;
+ Connector c = getConnector();
+ runMergeTest(c, "foo" + tc++, ns(), ns(), ns("l", "m", "n"), ns(null, "l"), ns(null, "n"));
+
+ runMergeTest(c, "foo" + tc++, ns("m"), ns(), ns("l", "m", "n"), ns(null, "l"), ns(null, "n"));
+ runMergeTest(c, "foo" + tc++, ns("m"), ns("m"), ns("l", "m", "n"), ns("m", "n"), ns(null, "z"));
+ runMergeTest(c, "foo" + tc++, ns("m"), ns("m"), ns("l", "m", "n"), ns(null, "b"), ns("l", "m"));
+
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns(), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns(null, "a"), ns(null, "s"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("m", "r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns(null, "a"), ns("c", "m"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns(null, "a"), ns("n", "r"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns("b", "c"), ns(null, "s"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b", "m"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns("m", "n"), ns(null, "s"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b", "r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns("b", "c"), ns("q", "r"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b", "m", "r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns(null, "a"), ns("aa", "b"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b", "m", "r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns("r", "s"), ns(null, "z"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b", "m", "r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns("b", "c"), ns("l", "m"));
+ runMergeTest(c, "foo" + tc++, ns("b", "m", "r"), ns("b", "m", "r"), ns("a", "b", "c", "l", "m", "n", "q", "r", "s"), ns("m", "n"), ns("q", "r"));
+
+ }
+
+ private void runMergeTest(Connector c, String table, String[] splits, String[] expectedSplits, String[] inserts, String[] start, String[] end) throws Exception {
+ int count = 0;
+
+ for (String s : start) {
+ for (String e : end) {
+ runMergeTest(c, table + "_" + count++, splits, expectedSplits, inserts, s, e);
+ }
+ }
+ }
+
+ private void runMergeTest(Connector conn, String table, String[] splits, String[] expectedSplits, String[] inserts, String start, String end) throws Exception {
+ System.out.println("Running merge test " + table + " " + Arrays.asList(splits) + " " + start + " " + end);
+
+ conn.tableOperations().create(table, true, TimeType.LOGICAL);
+ TreeSet<Text> splitSet = new TreeSet<Text>();
+ for (String split : splits) {
+ splitSet.add(new Text(split));
+ }
+ conn.tableOperations().addSplits(table, splitSet);
+
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ HashSet<String> expected = new HashSet<String>();
+ for (String row : inserts) {
+ Mutation m = new Mutation(row);
+ m.put("cf", "cq", row);
+ bw.addMutation(m);
+ expected.add(row);
+ }
+
+ bw.close();
+
+ conn.tableOperations().merge(table, start == null ? null : new Text(start), end == null ? null : new Text(end));
+
+ Scanner scanner = conn.createScanner(table, Authorizations.EMPTY);
+
+ HashSet<String> observed = new HashSet<String>();
+ for (Entry<Key,Value> entry : scanner) {
+ String row = entry.getKey().getRowData().toString();
+ if (!observed.add(row)) {
+ throw new Exception("Saw data twice " + table + " " + row);
+ }
+ }
+
+ if (!observed.equals(expected)) {
+ throw new Exception("data inconsistency " + table + " " + observed + " != " + expected);
+ }
+
+ HashSet<Text> currentSplits = new HashSet<Text>(conn.tableOperations().listSplits(table));
+ HashSet<Text> ess = new HashSet<Text>();
+ for (String es : expectedSplits) {
+ ess.add(new Text(es));
+ }
+
+ if (!currentSplits.equals(ess)) {
+ throw new Exception("split inconsistency " + table + " " + currentSplits + " != " + ess);
+ }
+
+ }
+
+}
Propchange: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeMetaIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeMetaIT.java?rev=1499110&view=auto
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeMetaIT.java (added)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeMetaIT.java Tue Jul 2 20:57:21 2013
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.*;
+
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.MetadataTable;
+import org.apache.accumulo.core.util.RootTable;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class MergeMetaIT extends MacTest {
+
+ @Test(timeout=30*1000)
+ public void mergeMeta() throws Exception {
+ Connector c = getConnector();
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (String id : "1 2 3 4 5".split(" ")) {
+ splits.add(new Text(id));
+ }
+ c.tableOperations().addSplits(MetadataTable.NAME, splits);
+ for (String tableName : "a1 a2 a3 a4 a5".split(" ")) {
+ c.tableOperations().create(tableName);
+ }
+ c.tableOperations().merge(MetadataTable.NAME, null, null);
+ UtilWaitThread.sleep(2*1000);
+ Scanner s = c.createScanner(RootTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataTable.DELETED_RANGE);
+ int count = 0;
+ for (@SuppressWarnings("unused") Entry<Key,Value> e : s) {
+ count++;
+ }
+ assertTrue(count > 0);
+ assertEquals(0, c.tableOperations().listSplits(MetadataTable.NAME).size());
+ }
+
+}
Propchange: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/functional/MergeMetaIT.java
------------------------------------------------------------------------------
svn:eol-style = native