You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/11/25 21:25:18 UTC
[3/4] git commit: ACCUMULO-1878 fix example integration tests
ACCUMULO-1878 fix example integration tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fd652ca0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fd652ca0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fd652ca0
Branch: refs/heads/master
Commit: fd652ca0dd60030b6b7e665476ad2ed72574dd36
Parents: 9196910 c175bd3
Author: Eric Newton <er...@gmail.com>
Authored: Mon Nov 25 15:25:12 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 15:25:12 2013 -0500
----------------------------------------------------------------------
.../simple/client/RandomBatchScanner.java | 21 +++++++++++++------
.../mapreduce/bulk/BulkIngestExample.java | 2 +-
.../simple/mapreduce/bulk/GenerateTestData.java | 2 +-
.../accumulo/test/functional/ExamplesIT.java | 22 +++++++-------------
4 files changed, 25 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fd652ca0/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
index 2b3289e,f75f11d..5f9b975
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/BulkIngestExample.java
@@@ -85,7 -84,7 +85,7 @@@ public class BulkIngestExample extends
int index = 0;
for (Text value : values) {
-- Key outputKey = new Key(key, new Text("foo"), new Text("" + index), timestamp);
++ Key outputKey = new Key(key, new Text("colf"), new Text(String.format("col_%07d", index)), timestamp);
index++;
Value outputValue = new Value(value.getBytes(), 0, value.getLength());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fd652ca0/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
index 9748d06,9748d06..c1a13b3
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/bulk/GenerateTestData.java
@@@ -45,7 -45,7 +45,7 @@@ public class GenerateTestData
PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(opts.outputFile))));
for (int i = 0; i < opts.numRows; i++) {
-- out.println(String.format("row_%08d\tvalue_%08d", i + opts.startRow, i + opts.startRow));
++ out.println(String.format("row_%010d\tvalue_%010d", i + opts.startRow, i + opts.startRow));
}
out.close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fd652ca0/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
index 459edf3,0000000..84803f1
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ExamplesIT.java
@@@ -1,291 -1,0 +1,285 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.client.Flush;
+import org.apache.accumulo.examples.simple.client.RandomBatchScanner;
+import org.apache.accumulo.examples.simple.client.RandomBatchWriter;
+import org.apache.accumulo.examples.simple.client.ReadWriteExample;
+import org.apache.accumulo.examples.simple.client.RowOperations;
+import org.apache.accumulo.examples.simple.client.SequentialBatchWriter;
+import org.apache.accumulo.examples.simple.client.TraceDumpExample;
+import org.apache.accumulo.examples.simple.client.TracingExample;
+import org.apache.accumulo.examples.simple.constraints.MaxMutationSize;
+import org.apache.accumulo.examples.simple.dirlist.Ingest;
+import org.apache.accumulo.examples.simple.dirlist.QueryUtil;
+import org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter;
+import org.apache.accumulo.examples.simple.helloworld.ReadData;
+import org.apache.accumulo.examples.simple.isolation.InterferenceTest;
+import org.apache.accumulo.examples.simple.mapreduce.RegexExample;
+import org.apache.accumulo.examples.simple.mapreduce.RowHash;
+import org.apache.accumulo.examples.simple.mapreduce.TableToFile;
+import org.apache.accumulo.examples.simple.mapreduce.TeraSortIngest;
+import org.apache.accumulo.examples.simple.mapreduce.WordCount;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData;
+import org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable;
+import org.apache.accumulo.examples.simple.shard.ContinuousQuery;
+import org.apache.accumulo.examples.simple.shard.Index;
+import org.apache.accumulo.examples.simple.shard.Query;
+import org.apache.accumulo.examples.simple.shard.Reverse;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster.LogWriter;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class ExamplesIT extends ConfigurableMacIT {
+
+ BatchWriterOpts bwOpts = new BatchWriterOpts();
+
+ @Override
+ public void configure(MiniAccumuloConfig cfg) {
+ cfg.setDefaultMemory(cfg.getDefaultMemory() * 2, MemoryUnit.BYTE);
+ }
+
+ @Test(timeout = 10 * 60 * 1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String instance = c.getInstance().getInstanceName();
+ String keepers = c.getInstance().getZooKeepers();
+ String user = "root";
+ String passwd = ROOT_PASSWORD;
+ String visibility = "A|B";
+ String auths = "A,B";
+ BatchWriterConfig bwc = new BatchWriterConfig();
+ BatchWriter bw;
+ IteratorSetting is;
+ String dir = cluster.getConfig().getDir().getAbsolutePath();
+ FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+
+ Process trace = cluster.exec(TraceServer.class);
+ while (!c.tableOperations().exists("trace"))
+ UtilWaitThread.sleep(500);
+
+ log.info("trace example");
+ Process p = cluster.exec(TracingExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-C", "-D", "-c");
+ assertEquals(0, p.waitFor());
+ for (LogWriter writer : cluster.getLogWriters()) {
+ writer.flush();
+ }
+ String result = FunctionalTestUtils.readAll(cluster, TracingExample.class, p);
+ Pattern pattern = Pattern.compile("TraceID: ([0-9a-f]+)");
+ Matcher matcher = pattern.matcher(result);
+ int count = 0;
+ while (matcher.find()) {
+ p = cluster.exec(TraceDumpExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--traceid", matcher.group(1));
+ assertEquals(0, p.waitFor());
+ count++;
+ }
+ assertTrue(count > 0);
+ result = FunctionalTestUtils.readAll(cluster, TraceDumpExample.class, p);
+ assertTrue(result.contains("myHost@myApp"));
+ trace.destroy();
+
+ log.info("testing dirlist example (a little)");
+ c.securityOperations().changeUserAuthorizations(user, new Authorizations(auths.split(",")));
+ assertEquals(
+ 0,
+ cluster.exec(Ingest.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--dirTable", "dirTable", "--indexTable", "indexTable",
+ "--dataTable", "dataTable", "--vis", visibility, "--chunkSize", 10000 + "", cluster.getConfig().getDir().getAbsolutePath()).waitFor());
+ p = cluster.exec(QueryUtil.class, "-i", instance, "-z", keepers, "-p", passwd, "-u", user, "-t", "indexTable", "--auths", auths, "--search", "--path",
+ "accumulo-site.xml");
+ assertEquals(0, p.waitFor());
+ for (LogWriter writer : cluster.getLogWriters()) {
+ writer.flush();
+ }
+ result = FunctionalTestUtils.readAll(cluster, QueryUtil.class, p);
+ System.out.println("result " + result);
+ assertTrue(result.contains("accumulo-site.xml"));
+
+ log.info("Testing ageoff filtering");
+ c.tableOperations().create("filtertest");
+ is = new IteratorSetting(10, AgeOffFilter.class);
+ AgeOffFilter.setTTL(is, 1000L);
+ c.tableOperations().attachIterator("filtertest", is);
+ bw = c.createBatchWriter("filtertest", bwc);
+ Mutation m = new Mutation("foo");
+ m.put("a", "b", "c");
+ bw.addMutation(m);
+ UtilWaitThread.sleep(1000);
+ count = 0;
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> line : c.createScanner("filtertest", Authorizations.EMPTY))
+ count++;
+ assertEquals(0, count);
+
+ log.info("Testing bloom filters are fast for missing data");
+ c.tableOperations().create("bloom_test");
+ c.tableOperations().setProperty("bloom_test", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+ assertEquals(
+ 0,
+ cluster.exec(RandomBatchWriter.class, "--seed", "7", "-i", instance, "-z", keepers, "-u", user, "-p", ROOT_PASSWORD, "--num", "100000", "--min", "0",
- "--max", "1000000000", "--size", "50", "--batchMemmory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", "bloom_test").waitFor());
++ "--max", "1000000000", "--size", "50", "--batchMemory", "2M", "--batchLatency", "60s", "--batchThreads", "3", "-t", "bloom_test").waitFor());
+ c.tableOperations().flush("bloom_test", null, null, true);
+ long diff = 0, diff2 = 0;
+ // try the speed test a couple times in case the system is loaded with other tests
+ for (int i = 0; i < 2; i++) {
+ long now = System.currentTimeMillis();
+ assertEquals(0, cluster.exec(RandomBatchScanner.class,"--seed", "7", "-i", instance, "-z",
+ keepers, "-u", user, "-p", ROOT_PASSWORD, "--num", "10000", "--min", "0", "--max", "1000000000", "--size", "50",
+ "--scanThreads", "4","-t", "bloom_test").waitFor());
+ diff = System.currentTimeMillis() - now;
+ now = System.currentTimeMillis();
- assertEquals(0, cluster.exec(RandomBatchScanner.class,"--seed", "8", "-i", instance, "-z",
++ assertEquals(1, cluster.exec(RandomBatchScanner.class,"--seed", "8", "-i", instance, "-z",
+ keepers, "-u", user, "-p", ROOT_PASSWORD, "--num", "10000", "--min", "0", "--max", "1000000000", "--size", "50",
+ "--scanThreads", "4","-t", "bloom_test").waitFor());
+ diff2 = System.currentTimeMillis() - now;
+ if (diff2 < diff)
+ break;
+ }
+ assertTrue(diff2 < diff);
+
+ log.info("Creating a sharded index of the accumulo java files");
+ c.tableOperations().create("shard");
+ c.tableOperations().create("doc2term");
+ bw = c.createBatchWriter("shard", bwc);
+ Index.index(30, new File(System.getProperty("user.dir") + "/src"), "\\W+", bw);
+ bw.close();
+ BatchScanner bs = c.createBatchScanner("shard", Authorizations.EMPTY, 4);
+ List<String> found = Query.query(bs, Arrays.asList("foo", "bar"));
+ bs.close();
+ // should find ourselves
+ boolean thisFile = false;
+ for (String file : found) {
+ if (file.endsWith("/ExamplesIT.java"))
+ thisFile = true;
+ }
+ assertTrue(thisFile);
+ // create a reverse index
- assertEquals(0, cluster.exec(Reverse.class, "-i", instance, "-z", keepers, "-t", "shard", "--doc2Term", "-u", "root", "-p", passwd).waitFor());
++ c.tableOperations().create("doc2Term");
++ assertEquals(0, cluster.exec(Reverse.class, "-i", instance, "-z", keepers, "--shardTable", "shard", "--doc2Term", "doc2Term", "-u", "root", "-p", passwd).waitFor());
+ // run some queries
+ assertEquals(
+ 0,
- cluster.exec(ContinuousQuery.class, "-i", instance, "-z", keepers, "-t", "shard", "--doc2Term", "-u", "root", "-p", passwd, "--term", "5", "--count",
++ cluster.exec(ContinuousQuery.class, "-i", instance, "-z", keepers, "--shardTable", "shard", "--doc2Term", "doc2Term", "-u", "root", "-p", passwd, "--terms", "5", "--count",
+ "1000").waitFor());
+
+ log.info("Testing MaxMutation constraint");
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().addConstraint("test_ingest", MaxMutationSize.class.getName());
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.rows = 1;
+ opts.cols = 1000;
+ try {
+ TestIngest.ingest(c, opts, bwOpts);
+ } catch (MutationsRejectedException ex) {
+ assertEquals(1, ex.getConstraintViolationSummaries().size());
+ }
+
- log.info("Starting build ingest example");
- assertEquals(0, cluster.exec(GenerateTestData.class, "0", "10000", dir + "/tmp/input/data").waitFor());
- assertEquals(0, cluster.exec(SetupTable.class, instance, keepers, user, passwd, "bulkTable").waitFor());
- assertEquals(0, cluster.exec(BulkIngestExample.class, instance, keepers, user, passwd, "bulkTable", dir + "/tmp/input", dir + "/tmp").waitFor());
- assertEquals(0, cluster.exec(VerifyIngest.class, instance, keepers, user, passwd, "bulkTable", "0", "1000000").waitFor());
-
+ log.info("Starting bulk ingest example");
- assertEquals(0, cluster.exec(GenerateTestData.class, "0", "1000000", dir + "/tmp/input/data").waitFor());
- assertEquals(0, cluster.exec(SetupTable.class, instance, keepers, user, passwd, "bulkTable").waitFor());
- assertEquals(0, cluster.exec(BulkIngestExample.class, instance, keepers, user, passwd, "bulkTable", dir + "/tmp/input", dir + "/tmp").waitFor());
- assertEquals(0, cluster.exec(VerifyIngest.class, instance, keepers, user, passwd, "bulkTable", "0", "1000000").waitFor());
++ assertEquals(0, cluster.exec(GenerateTestData.class, "--start-row", "0", "--count", "10000", "--output", dir + "/tmp/input/data").waitFor());
++ assertEquals(0, cluster.exec(SetupTable.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "bulkTable").waitFor());
++ assertEquals(0, cluster.exec(BulkIngestExample.class, "-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "bulkTable", "--inputDir", dir + "/tmp/input", "--workDir", dir + "/tmp").waitFor());
+
+ log.info("Running TeraSortIngest example");
+ exec(TeraSortIngest.class, new String[] {"--count", (1000 * 1000) + "", "-nk", "10", "-xk", "10", "-nv", "10", "-xv", "10", "-t", "sorted", "-i", instance,
+ "-z", keepers, "-u", user, "-p", passwd, "--splits", "4"});
+ log.info("Running Regex example");
+ exec(RegexExample.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "sorted", "--rowRegex", ".*999.*", "--output",
+ dir + "/tmp/nines"});
+ log.info("Running RowHash example");
+ exec(RowHash.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "sorted", "--column", "c:"});
+ log.info("Running TableToFile example");
+ exec(TableToFile.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "sorted", "--output", dir + "/tmp/tableFile"});
+
+ log.info("Running word count example");
+ c.tableOperations().create("wordCount");
+ is = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column(new Text("count"))));
+ SummingCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
+ c.tableOperations().attachIterator("wordCount", is);
+ fs.copyFromLocalFile(new Path(new Path(System.getProperty("user.dir")).getParent(), "README"), new Path(dir + "/tmp/wc/README"));
+ exec(WordCount.class, new String[] {"-i", instance, "-u", user, "-p", passwd, "-z", keepers, "--input", dir + "/tmp/wc", "-t", "wordCount"});
+
+ log.info("Inserting data with a batch writer");
+ exec(InsertWithBatchWriter.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "helloBatch"});
+ log.info("Reading data");
+ exec(ReadData.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "helloBatch"});
+ log.info("Running isolated scans");
+ exec(InterferenceTest.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "itest1", "--iterations", "100000", "--isolated"});
+ log.info("Running scans without isolation");
+ exec(InterferenceTest.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "itest2", "--iterations", "100000",});
+ log.info("Performing some row operations");
+ exec(RowOperations.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd,});
+ log.info("Using the batch writer");
+ c.tableOperations().create("test");
+ exec(SequentialBatchWriter.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "-t", "test", "--start", "0", "--num", "100000",
+ "--size", "50", "--batchMemory", "10000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility});
+
+ log.info("Reading and writing some data");
+ exec(ReadWriteExample.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", "test2", "--createtable",
+ "-c", "--debug"});
+ log.info("Deleting some data");
+ exec(ReadWriteExample.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--auths", auths, "--table", "test2", "-d", "--debug"});
+ log.info("Writing some data with the batch writer");
+ c.tableOperations().create("test3");
+ exec(RandomBatchWriter.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "test3", "--num", "100000", "--min", "0",
+ "--max", "99999", "--size", "100", "--batchMemory", "1000000", "--batchLatency", "1000", "--batchThreads", "4", "--vis", visibility});
+ log.info("Reading some data with the batch scanner");
+ exec(RandomBatchScanner.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "test3", "--num", "10000", "--min", "0",
+ "--max", "99999", "--size", "100", "--scanThreads", "4", "--auths", auths});
+ log.info("Running an example table operation (Flush)");
+ exec(Flush.class, new String[] {"-i", instance, "-z", keepers, "-u", user, "-p", passwd, "--table", "test3",});
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+
+ }
+
+}