You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2016/12/09 17:16:48 UTC
[2/7] accumulo git commit: ACCUMULO-4511 Removed Accumulo Examples
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java
deleted file mode 100644
index ba1e32e..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Index.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.shard;
-
-import java.io.File;
-import java.io.FileReader;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * This program indexes a set of documents given on the command line into a shard table.
- *
- * What it writes to the table is row = partition id, column family = term, column qualifier = document id.
- *
- * See docs/examples/README.shard for instructions.
- */
-
-public class Index {
-
- static Text genPartition(int partition) {
- return new Text(String.format("%08x", Math.abs(partition)));
- }
-
- public static void index(int numPartitions, Text docId, String doc, String splitRegex, BatchWriter bw) throws Exception {
-
- String[] tokens = doc.split(splitRegex);
-
- Text partition = genPartition(doc.hashCode() % numPartitions);
-
- Mutation m = new Mutation(partition);
-
- HashSet<String> tokensSeen = new HashSet<>();
-
- for (String token : tokens) {
- token = token.toLowerCase();
-
- if (!tokensSeen.contains(token)) {
- tokensSeen.add(token);
- m.put(new Text(token), docId, new Value(new byte[0]));
- }
- }
-
- if (m.size() > 0)
- bw.addMutation(m);
- }
-
- public static void index(int numPartitions, File src, String splitRegex, BatchWriter bw) throws Exception {
- if (src.isDirectory()) {
- File[] files = src.listFiles();
- if (files != null) {
- for (File child : files) {
- index(numPartitions, child, splitRegex, bw);
- }
- }
- } else {
- FileReader fr = new FileReader(src);
-
- StringBuilder sb = new StringBuilder();
-
- char data[] = new char[4096];
- int len;
- while ((len = fr.read(data)) != -1) {
- sb.append(data, 0, len);
- }
-
- fr.close();
-
- index(numPartitions, new Text(src.getAbsolutePath()), sb.toString(), splitRegex, bw);
- }
-
- }
-
- static class Opts extends ClientOnRequiredTable {
- @Parameter(names = "--partitions", required = true, description = "the number of shards to create")
- int partitions;
- @Parameter(required = true, description = "<file> { <file> ... }")
- List<String> files = new ArrayList<>();
- }
-
- public static void main(String[] args) throws Exception {
- Opts opts = new Opts();
- BatchWriterOpts bwOpts = new BatchWriterOpts();
- opts.parseArgs(Index.class.getName(), args, bwOpts);
-
- String splitRegex = "\\W+";
-
- BatchWriter bw = opts.getConnector().createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
- for (String filename : opts.files) {
- index(opts.partitions, new File(filename), splitRegex, bw);
- }
- bw.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
deleted file mode 100644
index 13adcca..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.shard;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.cli.BatchScannerOpts;
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.sample.SamplerConfiguration;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.IntersectingIterator;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * This program queries a set of terms in the shard table (populated by {@link Index}) using the {@link IntersectingIterator}.
- *
- * See docs/examples/README.shard for instructions.
- */
-
-public class Query {
-
- static class Opts extends ClientOnRequiredTable {
- @Parameter(description = " term { <term> ... }")
- List<String> terms = new ArrayList<>();
-
- @Parameter(names = {"--sample"}, description = "Do queries against sample, useful when sample is built using column qualifier")
- private boolean useSample = false;
-
- @Parameter(names = {"--sampleCutoff"},
- description = "Use sample data to determine if a query might return a number of documents over the cutoff. This check is per tablet.")
- private Integer sampleCutoff = null;
- }
-
- public static List<String> query(BatchScanner bs, List<String> terms, Integer cutoff) {
-
- Text columns[] = new Text[terms.size()];
- int i = 0;
- for (String term : terms) {
- columns[i++] = new Text(term);
- }
-
- IteratorSetting ii;
-
- if (cutoff != null) {
- ii = new IteratorSetting(20, "ii", CutoffIntersectingIterator.class);
- CutoffIntersectingIterator.setCutoff(ii, cutoff);
- } else {
- ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
- }
-
- IntersectingIterator.setColumnFamilies(ii, columns);
- bs.addScanIterator(ii);
- bs.setRanges(Collections.singleton(new Range()));
- List<String> result = new ArrayList<>();
- for (Entry<Key,Value> entry : bs) {
- result.add(entry.getKey().getColumnQualifier().toString());
- }
- return result;
- }
-
- public static void main(String[] args) throws Exception {
- Opts opts = new Opts();
- BatchScannerOpts bsOpts = new BatchScannerOpts();
- opts.parseArgs(Query.class.getName(), args, bsOpts);
- Connector conn = opts.getConnector();
- BatchScanner bs = conn.createBatchScanner(opts.getTableName(), opts.auths, bsOpts.scanThreads);
- bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS);
- if (opts.useSample) {
- SamplerConfiguration samplerConfig = conn.tableOperations().getSamplerConfiguration(opts.getTableName());
- CutoffIntersectingIterator.validateSamplerConfig(conn.tableOperations().getSamplerConfiguration(opts.getTableName()));
- bs.setSamplerConfiguration(samplerConfig);
- }
- for (String entry : query(bs, opts.terms, opts.sampleCutoff))
- System.out.println(" " + entry);
-
- bs.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java
deleted file mode 100644
index dbcbe5f..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Reverse.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.shard;
-
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ClientOpts;
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * The program reads an accumulo table written by {@link Index} and writes out to another table. It writes out a mapping of documents to terms. The document to
- * term mapping is used by {@link ContinuousQuery}.
- *
- * See docs/examples/README.shard for instructions.
- */
-
-public class Reverse {
-
- static class Opts extends ClientOpts {
- @Parameter(names = "--shardTable")
- String shardTable = "shard";
- @Parameter(names = "--doc2Term")
- String doc2TermTable = "doc2Term";
- }
-
- public static void main(String[] args) throws Exception {
- Opts opts = new Opts();
- ScannerOpts scanOpts = new ScannerOpts();
- BatchWriterOpts bwOpts = new BatchWriterOpts();
- opts.parseArgs(Reverse.class.getName(), args, scanOpts, bwOpts);
-
- Connector conn = opts.getConnector();
-
- Scanner scanner = conn.createScanner(opts.shardTable, opts.auths);
- scanner.setBatchSize(scanOpts.scanBatchSize);
- BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, bwOpts.getBatchWriterConfig());
-
- for (Entry<Key,Value> entry : scanner) {
- Key key = entry.getKey();
- Mutation m = new Mutation(key.getColumnQualifier());
- m.put(key.getColumnFamily(), new Text(), new Value(new byte[0]));
- bw.addMutation(m);
- }
-
- bw.close();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/DebugCommand.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/DebugCommand.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/DebugCommand.java
deleted file mode 100644
index 4395fe7..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/DebugCommand.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.shell;
-
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.accumulo.shell.Shell;
-import org.apache.accumulo.shell.Shell.Command;
-import org.apache.commons.cli.CommandLine;
-
-public class DebugCommand extends Command {
-
- @Override
- public int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception {
- Set<String> lines = new TreeSet<>();
- lines.add("This is a test");
- shellState.printLines(lines.iterator(), true);
- return 0;
- }
-
- @Override
- public String description() {
- return "prints a message to test extension feature";
- }
-
- @Override
- public int numArgs() {
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/ExampleShellExtension.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/ExampleShellExtension.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/ExampleShellExtension.java
deleted file mode 100644
index bcd6690..0000000
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shell/ExampleShellExtension.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.shell;
-
-import org.apache.accumulo.shell.Shell.Command;
-import org.apache.accumulo.shell.ShellExtension;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(ShellExtension.class)
-public class ExampleShellExtension extends ShellExtension {
-
- @Override
- public String getExtensionName() {
- return "ExampleShellExtension";
- }
-
- @Override
- public Command[] getCommands() {
- return new Command[] {new DebugCommand()};
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraintTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraintTest.java b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraintTest.java
deleted file mode 100644
index 8ef3f0f..0000000
--- a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/AlphaNumKeyConstraintTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.constraints;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableList;
-
-public class AlphaNumKeyConstraintTest {
-
- private AlphaNumKeyConstraint ankc = new AlphaNumKeyConstraint();
-
- @Test
- public void test() {
- Mutation goodMutation = new Mutation(new Text("Row1"));
- goodMutation.put(new Text("Colf2"), new Text("ColQ3"), new Value("value".getBytes()));
- assertNull(ankc.check(null, goodMutation));
-
- // Check that violations are in row, cf, cq order
- Mutation badMutation = new Mutation(new Text("Row#1"));
- badMutation.put(new Text("Colf$2"), new Text("Colq%3"), new Value("value".getBytes()));
- assertEquals(ImmutableList.of(AlphaNumKeyConstraint.NON_ALPHA_NUM_ROW, AlphaNumKeyConstraint.NON_ALPHA_NUM_COLF, AlphaNumKeyConstraint.NON_ALPHA_NUM_COLQ),
- ankc.check(null, badMutation));
- }
-
- @Test
- public void testGetViolationDescription() {
- assertEquals(AlphaNumKeyConstraint.ROW_VIOLATION_MESSAGE, ankc.getViolationDescription(AlphaNumKeyConstraint.NON_ALPHA_NUM_ROW));
- assertEquals(AlphaNumKeyConstraint.COLF_VIOLATION_MESSAGE, ankc.getViolationDescription(AlphaNumKeyConstraint.NON_ALPHA_NUM_COLF));
- assertEquals(AlphaNumKeyConstraint.COLQ_VIOLATION_MESSAGE, ankc.getViolationDescription(AlphaNumKeyConstraint.NON_ALPHA_NUM_COLQ));
- assertNull(ankc.getViolationDescription((short) 4));
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraintTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraintTest.java b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraintTest.java
deleted file mode 100644
index 7d1fc49..0000000
--- a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/constraints/NumericValueConstraintTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.constraints;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-
-import com.google.common.collect.Iterables;
-
-public class NumericValueConstraintTest {
-
- private NumericValueConstraint nvc = new NumericValueConstraint();
-
- @Test
- public void testCheck() {
- Mutation goodMutation = new Mutation(new Text("r"));
- goodMutation.put(new Text("cf"), new Text("cq"), new Value("1234".getBytes()));
- assertNull(nvc.check(null, goodMutation));
-
- // Check that multiple bad mutations result in one violation only
- Mutation badMutation = new Mutation(new Text("r"));
- badMutation.put(new Text("cf"), new Text("cq"), new Value("foo1234".getBytes()));
- badMutation.put(new Text("cf2"), new Text("cq2"), new Value("foo1234".getBytes()));
- assertEquals(NumericValueConstraint.NON_NUMERIC_VALUE, Iterables.getOnlyElement(nvc.check(null, badMutation)).shortValue());
- }
-
- @Test
- public void testGetViolationDescription() {
- assertEquals(NumericValueConstraint.VIOLATION_MESSAGE, nvc.getViolationDescription(NumericValueConstraint.NON_NUMERIC_VALUE));
- assertNull(nvc.getViolationDescription((short) 2));
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkCombinerTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkCombinerTest.java b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkCombinerTest.java
deleted file mode 100644
index 40f4bb9..0000000
--- a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkCombinerTest.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import junit.framework.TestCase;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-
-public class ChunkCombinerTest extends TestCase {
-
- public static class MapIterator implements SortedKeyValueIterator<Key,Value> {
- private Iterator<Entry<Key,Value>> iter;
- private Entry<Key,Value> entry;
- Collection<ByteSequence> columnFamilies;
- private SortedMap<Key,Value> map;
- private Range range;
-
- @Override
- public MapIterator deepCopy(IteratorEnvironment env) {
- return new MapIterator(map);
- }
-
- private MapIterator(SortedMap<Key,Value> map) {
- this.map = map;
- iter = map.entrySet().iterator();
- this.range = new Range();
- if (iter.hasNext())
- entry = iter.next();
- else
- entry = null;
- }
-
- @Override
- public Key getTopKey() {
- return entry.getKey();
- }
-
- @Override
- public Value getTopValue() {
- return entry.getValue();
- }
-
- @Override
- public boolean hasTop() {
- return entry != null;
- }
-
- @Override
- public void next() throws IOException {
- entry = null;
- while (iter.hasNext()) {
- entry = iter.next();
- if (columnFamilies.size() > 0 && !columnFamilies.contains(entry.getKey().getColumnFamilyData())) {
- entry = null;
- continue;
- }
- if (range.afterEndKey(entry.getKey()))
- entry = null;
- break;
- }
- }
-
- @Override
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- if (!inclusive) {
- throw new IllegalArgumentException("can only do inclusive colf filtering");
- }
- this.columnFamilies = columnFamilies;
- this.range = range;
-
- Key key = range.getStartKey();
- if (key == null) {
- key = new Key();
- }
-
- iter = map.tailMap(key).entrySet().iterator();
- next();
- while (hasTop() && range.beforeStartKey(getTopKey())) {
- next();
- }
- }
-
- @Override
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- throw new UnsupportedOperationException();
- }
- }
-
- private TreeMap<Key,Value> row1;
- private TreeMap<Key,Value> row2;
- private TreeMap<Key,Value> row3;
- private TreeMap<Key,Value> allRows;
-
- private TreeMap<Key,Value> cRow1;
- private TreeMap<Key,Value> cRow2;
- private TreeMap<Key,Value> cRow3;
- private TreeMap<Key,Value> allCRows;
-
- private TreeMap<Key,Value> cOnlyRow1;
- private TreeMap<Key,Value> cOnlyRow2;
- private TreeMap<Key,Value> cOnlyRow3;
- private TreeMap<Key,Value> allCOnlyRows;
-
- private TreeMap<Key,Value> badrow;
-
- @Override
- protected void setUp() {
- row1 = new TreeMap<>();
- row2 = new TreeMap<>();
- row3 = new TreeMap<>();
- allRows = new TreeMap<>();
-
- cRow1 = new TreeMap<>();
- cRow2 = new TreeMap<>();
- cRow3 = new TreeMap<>();
- allCRows = new TreeMap<>();
-
- cOnlyRow1 = new TreeMap<>();
- cOnlyRow2 = new TreeMap<>();
- cOnlyRow3 = new TreeMap<>();
- allCOnlyRows = new TreeMap<>();
-
- badrow = new TreeMap<>();
-
- String refs = FileDataIngest.REFS_CF.toString();
- String fileext = FileDataIngest.REFS_FILE_EXT;
- String filename = FileDataIngest.REFS_ORIG_FILE;
- String chunk_cf = FileDataIngest.CHUNK_CF.toString();
-
- row1.put(new Key("row1", refs, "hash1\0" + fileext, "C"), new Value("jpg".getBytes()));
- row1.put(new Key("row1", refs, "hash1\0" + filename, "D"), new Value("foo1.jpg".getBytes()));
- row1.put(new Key("row1", chunk_cf, "0000", "A"), new Value("V1".getBytes()));
- row1.put(new Key("row1", chunk_cf, "0000", "B"), new Value("V1".getBytes()));
- row1.put(new Key("row1", chunk_cf, "0001", "A"), new Value("V2".getBytes()));
- row1.put(new Key("row1", chunk_cf, "0001", "B"), new Value("V2".getBytes()));
-
- cRow1.put(new Key("row1", refs, "hash1\0" + fileext, "C"), new Value("jpg".getBytes()));
- cRow1.put(new Key("row1", refs, "hash1\0" + filename, "D"), new Value("foo1.jpg".getBytes()));
- cRow1.put(new Key("row1", chunk_cf, "0000", "(C)|(D)"), new Value("V1".getBytes()));
- cRow1.put(new Key("row1", chunk_cf, "0001", "(C)|(D)"), new Value("V2".getBytes()));
-
- cOnlyRow1.put(new Key("row1", chunk_cf, "0000", "(C)|(D)"), new Value("V1".getBytes()));
- cOnlyRow1.put(new Key("row1", chunk_cf, "0001", "(C)|(D)"), new Value("V2".getBytes()));
-
- row2.put(new Key("row2", refs, "hash1\0" + fileext, "A"), new Value("jpg".getBytes()));
- row2.put(new Key("row2", refs, "hash1\0" + filename, "B"), new Value("foo1.jpg".getBytes()));
- row2.put(new Key("row2", chunk_cf, "0000", "A|B"), new Value("V1".getBytes()));
- row2.put(new Key("row2", chunk_cf, "0000", "A"), new Value("V1".getBytes()));
- row2.put(new Key("row2", chunk_cf, "0000", "(A)|(B)"), new Value("V1".getBytes()));
- row2.put(new Key("row2a", chunk_cf, "0000", "C"), new Value("V1".getBytes()));
-
- cRow2.put(new Key("row2", refs, "hash1\0" + fileext, "A"), new Value("jpg".getBytes()));
- cRow2.put(new Key("row2", refs, "hash1\0" + filename, "B"), new Value("foo1.jpg".getBytes()));
- cRow2.put(new Key("row2", chunk_cf, "0000", "(A)|(B)"), new Value("V1".getBytes()));
-
- cOnlyRow2.put(new Key("row2", chunk_cf, "0000", "(A)|(B)"), new Value("V1".getBytes()));
-
- row3.put(new Key("row3", refs, "hash1\0w", "(A&B)|(C&(D|E))"), new Value("".getBytes()));
- row3.put(new Key("row3", refs, "hash1\0x", "A&B"), new Value("".getBytes()));
- row3.put(new Key("row3", refs, "hash1\0y", "(A&B)"), new Value("".getBytes()));
- row3.put(new Key("row3", refs, "hash1\0z", "(F|G)&(D|E)"), new Value("".getBytes()));
- row3.put(new Key("row3", chunk_cf, "0000", "(A&B)|(C&(D|E))", 10), new Value("V1".getBytes()));
- row3.put(new Key("row3", chunk_cf, "0000", "A&B", 20), new Value("V1".getBytes()));
- row3.put(new Key("row3", chunk_cf, "0000", "(A&B)", 10), new Value("V1".getBytes()));
- row3.put(new Key("row3", chunk_cf, "0000", "(F|G)&(D|E)", 10), new Value("V1".getBytes()));
-
- cRow3.put(new Key("row3", refs, "hash1\0w", "(A&B)|(C&(D|E))"), new Value("".getBytes()));
- cRow3.put(new Key("row3", refs, "hash1\0x", "A&B"), new Value("".getBytes()));
- cRow3.put(new Key("row3", refs, "hash1\0y", "(A&B)"), new Value("".getBytes()));
- cRow3.put(new Key("row3", refs, "hash1\0z", "(F|G)&(D|E)"), new Value("".getBytes()));
- cRow3.put(new Key("row3", chunk_cf, "0000", "((F|G)&(D|E))|(A&B)|(C&(D|E))", 20), new Value("V1".getBytes()));
-
- cOnlyRow3.put(new Key("row3", chunk_cf, "0000", "((F|G)&(D|E))|(A&B)|(C&(D|E))", 20), new Value("V1".getBytes()));
-
- badrow.put(new Key("row1", chunk_cf, "0000", "A"), new Value("V1".getBytes()));
- badrow.put(new Key("row1", chunk_cf, "0000", "B"), new Value("V2".getBytes()));
-
- allRows.putAll(row1);
- allRows.putAll(row2);
- allRows.putAll(row3);
-
- allCRows.putAll(cRow1);
- allCRows.putAll(cRow2);
- allCRows.putAll(cRow3);
-
- allCOnlyRows.putAll(cOnlyRow1);
- allCOnlyRows.putAll(cOnlyRow2);
- allCOnlyRows.putAll(cOnlyRow3);
- }
-
- private static final Collection<ByteSequence> emptyColfs = new HashSet<>();
-
- public void test1() throws IOException {
- runTest(false, allRows, allCRows, emptyColfs);
- runTest(true, allRows, allCRows, emptyColfs);
- runTest(false, allRows, allCOnlyRows, Collections.singleton(FileDataIngest.CHUNK_CF_BS));
- runTest(true, allRows, allCOnlyRows, Collections.singleton(FileDataIngest.CHUNK_CF_BS));
-
- try {
- runTest(true, badrow, null, emptyColfs);
- assertNotNull(null);
- } catch (RuntimeException e) {
- assertNull(null);
- }
- }
-
- private void runTest(boolean reseek, TreeMap<Key,Value> source, TreeMap<Key,Value> result, Collection<ByteSequence> cols) throws IOException {
- MapIterator src = new MapIterator(source);
- SortedKeyValueIterator<Key,Value> iter = new ChunkCombiner();
- iter.init(src, null, null);
- iter = iter.deepCopy(null);
- iter.seek(new Range(), cols, true);
-
- TreeMap<Key,Value> seen = new TreeMap<>();
-
- while (iter.hasTop()) {
- assertFalse("already contains " + iter.getTopKey(), seen.containsKey(iter.getTopKey()));
- seen.put(new Key(iter.getTopKey()), new Value(iter.getTopValue()));
-
- if (reseek)
- iter.seek(new Range(iter.getTopKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL), true, null, true), cols, true);
- else
- iter.next();
- }
-
- assertEquals(result, seen);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStreamTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStreamTest.java b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStreamTest.java
deleted file mode 100644
index 2796d47..0000000
--- a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputStreamTest.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyValue;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.PeekingIterator;
-import org.apache.hadoop.io.Text;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ChunkInputStreamTest {
- private static final Logger log = LoggerFactory.getLogger(ChunkInputStream.class);
- private List<Entry<Key,Value>> data;
- private List<Entry<Key,Value>> baddata;
- private List<Entry<Key,Value>> multidata;
-
- @Before
- public void setupData() {
- data = new ArrayList<>();
- addData(data, "a", "refs", "id\0ext", "A&B", "ext");
- addData(data, "a", "refs", "id\0name", "A&B", "name");
- addData(data, "a", "~chunk", 100, 0, "A&B", "asdfjkl;");
- addData(data, "a", "~chunk", 100, 1, "A&B", "");
- addData(data, "b", "refs", "id\0ext", "A&B", "ext");
- addData(data, "b", "refs", "id\0name", "A&B", "name");
- addData(data, "b", "~chunk", 100, 0, "A&B", "qwertyuiop");
- addData(data, "b", "~chunk", 100, 0, "B&C", "qwertyuiop");
- addData(data, "b", "~chunk", 100, 1, "A&B", "");
- addData(data, "b", "~chunk", 100, 1, "B&C", "");
- addData(data, "b", "~chunk", 100, 1, "D", "");
- addData(data, "c", "~chunk", 100, 0, "A&B", "asdfjkl;");
- addData(data, "c", "~chunk", 100, 1, "A&B", "asdfjkl;");
- addData(data, "c", "~chunk", 100, 2, "A&B", "");
- addData(data, "d", "~chunk", 100, 0, "A&B", "");
- addData(data, "e", "~chunk", 100, 0, "A&B", "asdfjkl;");
- addData(data, "e", "~chunk", 100, 1, "A&B", "");
- baddata = new ArrayList<>();
- addData(baddata, "a", "~chunk", 100, 0, "A", "asdfjkl;");
- addData(baddata, "b", "~chunk", 100, 0, "B", "asdfjkl;");
- addData(baddata, "b", "~chunk", 100, 2, "C", "");
- addData(baddata, "c", "~chunk", 100, 0, "D", "asdfjkl;");
- addData(baddata, "c", "~chunk", 100, 2, "E", "");
- addData(baddata, "d", "~chunk", 100, 0, "F", "asdfjkl;");
- addData(baddata, "d", "~chunk", 100, 1, "G", "");
- addData(baddata, "d", "~zzzzz", "colq", "H", "");
- addData(baddata, "e", "~chunk", 100, 0, "I", "asdfjkl;");
- addData(baddata, "e", "~chunk", 100, 1, "J", "");
- addData(baddata, "e", "~chunk", 100, 2, "I", "asdfjkl;");
- addData(baddata, "f", "~chunk", 100, 2, "K", "asdfjkl;");
- addData(baddata, "g", "~chunk", 100, 0, "L", "");
- multidata = new ArrayList<>();
- addData(multidata, "a", "~chunk", 100, 0, "A&B", "asdfjkl;");
- addData(multidata, "a", "~chunk", 100, 1, "A&B", "");
- addData(multidata, "a", "~chunk", 200, 0, "B&C", "asdfjkl;");
- addData(multidata, "b", "~chunk", 100, 0, "A&B", "asdfjkl;");
- addData(multidata, "b", "~chunk", 200, 0, "B&C", "asdfjkl;");
- addData(multidata, "b", "~chunk", 200, 1, "B&C", "asdfjkl;");
- addData(multidata, "c", "~chunk", 100, 0, "A&B", "asdfjkl;");
- addData(multidata, "c", "~chunk", 100, 1, "B&C", "");
- }
-
- private static void addData(List<Entry<Key,Value>> data, String row, String cf, String cq, String vis, String value) {
- data.add(new KeyValue(new Key(new Text(row), new Text(cf), new Text(cq), new Text(vis)), value.getBytes()));
- }
-
- private static void addData(List<Entry<Key,Value>> data, String row, String cf, int chunkSize, int chunkCount, String vis, String value) {
- Text chunkCQ = new Text(FileDataIngest.intToBytes(chunkSize));
- chunkCQ.append(FileDataIngest.intToBytes(chunkCount), 0, 4);
- data.add(new KeyValue(new Key(new Text(row), new Text(cf), chunkCQ, new Text(vis)), value.getBytes()));
- }
-
- @Test
- public void testExceptionOnMultipleSetSourceWithoutClose() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(data.iterator());
- pi = new PeekingIterator<>(data.iterator());
- cis.setSource(pi);
- try {
- cis.setSource(pi);
- fail();
- } catch (IOException e) {
- /* expected */
- }
- cis.close();
- }
-
- @Test
- public void testExceptionOnGetVisBeforeClose() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(data.iterator());
-
- cis.setSource(pi);
- try {
- cis.getVisibilities();
- fail();
- } catch (RuntimeException e) {
- /* expected */
- }
- cis.close();
- cis.getVisibilities();
- }
-
- @Test
- public void testReadIntoBufferSmallerThanChunks() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- byte[] b = new byte[5];
-
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(data.iterator());
-
- cis.setSource(pi);
- int read;
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "asdfj");
- assertEquals(read = cis.read(b), 3);
- assertEquals(new String(b, 0, read), "kl;");
- assertEquals(read = cis.read(b), -1);
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "qwert");
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "yuiop");
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[A&B, B&C, D]");
- cis.close();
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "asdfj");
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "kl;as");
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "dfjkl");
- assertEquals(read = cis.read(b), 1);
- assertEquals(new String(b, 0, read), ";");
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[A&B]");
- cis.close();
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), -1);
- cis.close();
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 5);
- assertEquals(new String(b, 0, read), "asdfj");
- assertEquals(read = cis.read(b), 3);
- assertEquals(new String(b, 0, read), "kl;");
- assertEquals(read = cis.read(b), -1);
- cis.close();
-
- assertFalse(pi.hasNext());
- }
-
- @Test
- public void testReadIntoBufferLargerThanChunks() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- byte[] b = new byte[20];
- int read;
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(data.iterator());
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 8);
- assertEquals(new String(b, 0, read), "asdfjkl;");
- assertEquals(read = cis.read(b), -1);
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 10);
- assertEquals(new String(b, 0, read), "qwertyuiop");
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[A&B, B&C, D]");
- cis.close();
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 16);
- assertEquals(new String(b, 0, read), "asdfjkl;asdfjkl;");
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[A&B]");
- cis.close();
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), -1);
- cis.close();
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 8);
- assertEquals(new String(b, 0, read), "asdfjkl;");
- assertEquals(read = cis.read(b), -1);
- cis.close();
-
- assertFalse(pi.hasNext());
- }
-
- private static void assumeExceptionOnRead(ChunkInputStream cis, byte[] b) {
- try {
- assertEquals(0, cis.read(b));
- fail();
- } catch (IOException e) {
- log.debug("EXCEPTION {}", e.getMessage());
- // expected, ignore
- }
- }
-
- private static void assumeExceptionOnClose(ChunkInputStream cis) {
- try {
- cis.close();
- fail();
- } catch (IOException e) {
- log.debug("EXCEPTION {}", e.getMessage());
- // expected, ignore
- }
- }
-
- @Test
- public void testBadData() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- byte[] b = new byte[20];
- int read;
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(baddata.iterator());
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assumeExceptionOnClose(cis);
- // can still get visibilities after exception -- bad?
- assertEquals(cis.getVisibilities().toString(), "[A]");
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assumeExceptionOnClose(cis);
- assertEquals(cis.getVisibilities().toString(), "[B, C]");
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assumeExceptionOnClose(cis);
- assertEquals(cis.getVisibilities().toString(), "[D, E]");
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 8);
- assertEquals(new String(b, 0, read), "asdfjkl;");
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[F, G]");
- cis.close();
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- cis.close();
- assertEquals(cis.getVisibilities().toString(), "[I, J]");
-
- try {
- cis.setSource(pi);
- fail();
- } catch (IOException e) {
- // expected, ignore
- }
- assumeExceptionOnClose(cis);
- assertEquals(cis.getVisibilities().toString(), "[K]");
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[L]");
- cis.close();
-
- assertFalse(pi.hasNext());
-
- pi = new PeekingIterator<>(baddata.iterator());
- cis.setSource(pi);
- assumeExceptionOnClose(cis);
- }
-
- @Test
- public void testBadDataWithoutClosing() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- byte[] b = new byte[20];
- int read;
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(baddata.iterator());
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- // can still get visibilities after exception -- bad?
- assertEquals(cis.getVisibilities().toString(), "[A]");
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assertEquals(cis.getVisibilities().toString(), "[B, C]");
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assertEquals(cis.getVisibilities().toString(), "[D, E]");
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 8);
- assertEquals(new String(b, 0, read), "asdfjkl;");
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[F, G]");
- cis.close();
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assertEquals(cis.getVisibilities().toString(), "[I, J]");
-
- try {
- cis.setSource(pi);
- fail();
- } catch (IOException e) {
- // expected, ignore
- }
- assertEquals(cis.getVisibilities().toString(), "[K]");
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), -1);
- assertEquals(cis.getVisibilities().toString(), "[L]");
- cis.close();
-
- assertFalse(pi.hasNext());
-
- pi = new PeekingIterator<>(baddata.iterator());
- cis.setSource(pi);
- assumeExceptionOnClose(cis);
- }
-
- @Test
- public void testMultipleChunkSizes() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- byte[] b = new byte[20];
- int read;
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(multidata.iterator());
-
- b = new byte[20];
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 8);
- assertEquals(read = cis.read(b), -1);
- cis.close();
- assertEquals(cis.getVisibilities().toString(), "[A&B]");
-
- cis.setSource(pi);
- assumeExceptionOnRead(cis, b);
- assertEquals(cis.getVisibilities().toString(), "[A&B]");
-
- cis.setSource(pi);
- assertEquals(read = cis.read(b), 8);
- assertEquals(new String(b, 0, read), "asdfjkl;");
- assertEquals(read = cis.read(b), -1);
- cis.close();
- assertEquals(cis.getVisibilities().toString(), "[A&B, B&C]");
-
- assertFalse(pi.hasNext());
- }
-
- @Test
- public void testSingleByteRead() throws IOException {
- ChunkInputStream cis = new ChunkInputStream();
- PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<>(data.iterator());
-
- cis.setSource(pi);
- assertEquals((byte) 'a', (byte) cis.read());
- assertEquals((byte) 's', (byte) cis.read());
- assertEquals((byte) 'd', (byte) cis.read());
- assertEquals((byte) 'f', (byte) cis.read());
- assertEquals((byte) 'j', (byte) cis.read());
- assertEquals((byte) 'k', (byte) cis.read());
- assertEquals((byte) 'l', (byte) cis.read());
- assertEquals((byte) ';', (byte) cis.read());
- assertEquals(cis.read(), -1);
- cis.close();
- assertEquals(cis.getVisibilities().toString(), "[A&B]");
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/KeyUtilTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/KeyUtilTest.java b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/KeyUtilTest.java
deleted file mode 100644
index e93331a..0000000
--- a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/KeyUtilTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.simple.filedata;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.io.Text;
-
-public class KeyUtilTest extends TestCase {
- public static void checkSeps(String... s) {
- Text t = KeyUtil.buildNullSepText(s);
- String[] rets = KeyUtil.splitNullSepText(t);
-
- int length = 0;
- for (String str : s)
- length += str.length();
- assertEquals(t.getLength(), length + s.length - 1);
- assertEquals(rets.length, s.length);
- for (int i = 0; i < s.length; i++)
- assertEquals(s[i], rets[i]);
- }
-
- public void testNullSep() {
- checkSeps("abc", "d", "", "efgh");
- checkSeps("ab", "");
- checkSeps("abcde");
- checkSeps("");
- checkSeps("", "");
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/examples/simple/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/resources/log4j.properties b/examples/simple/src/test/resources/log4j.properties
deleted file mode 100644
index 133a28c..0000000
--- a/examples/simple/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, CA
-log4j.appender.CA=org.apache.log4j.ConsoleAppender
-log4j.appender.CA.layout=org.apache.log4j.PatternLayout
-log4j.appender.CA.layout.ConversionPattern=[%t} %-5p %c %x - %m%n
-
-log4j.logger.org.apache.hadoop.mapred=ERROR
-log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
-log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6280565..9128dd0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,7 +83,6 @@
<module>assemble</module>
<module>core</module>
<module>docs</module>
- <module>examples/simple</module>
<module>fate</module>
<module>iterator-test-harness</module>
<module>maven-plugin</module>
@@ -269,11 +268,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-examples-simple</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-fate</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/proxy/examples/python/TestNamespace.py
----------------------------------------------------------------------
diff --git a/proxy/examples/python/TestNamespace.py b/proxy/examples/python/TestNamespace.py
index e7d2377..1e3db5e 100644
--- a/proxy/examples/python/TestNamespace.py
+++ b/proxy/examples/python/TestNamespace.py
@@ -118,13 +118,13 @@ def main():
print 'adding max mutation size namespace constraint'
constraintid = client.addNamespaceConstraint(login, 'testing',
- 'org.apache.accumulo.examples.simple.constraints.MaxMutationSize')
+ 'org.apache.accumulo.test.constraints.MaxMutationSize')
print 'make sure constraint was added'
constraints = client.listNamespaceConstraints(login, 'testing')
found = False
for name, cid in constraints.iteritems():
- if cid == constraintid and name == 'org.apache.accumulo.examples.simple.constraints.MaxMutationSize':
+ if cid == constraintid and name == 'org.apache.accumulo.test.constraints.MaxMutationSize':
found = True
break
assert found
@@ -136,7 +136,7 @@ def main():
constraints = client.listNamespaceConstraints(login, 'testing')
found = False
for name, cid in constraints.iteritems():
- if cid == constraintid and name == 'org.apache.accumulo.examples.simple.constraints.MaxMutationSize':
+ if cid == constraintid and name == 'org.apache.accumulo.test.constraints.MaxMutationSize':
found = True
break
assert not found
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/proxy/pom.xml b/proxy/pom.xml
index 6bb4eb2..253c5e8 100644
--- a/proxy/pom.xml
+++ b/proxy/pom.xml
@@ -77,11 +77,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-examples-simple</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index 0bad02f..18a28dc 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -84,10 +84,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-examples-simple</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-fate</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java
index f0b46b5..7df5885 100644
--- a/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ConditionalWriterIT.java
@@ -87,7 +87,7 @@ import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.core.trace.Span;
import org.apache.accumulo.core.trace.Trace;
import org.apache.accumulo.core.util.FastFormat;
-import org.apache.accumulo.examples.simple.constraints.AlphaNumKeyConstraint;
+import org.apache.accumulo.test.constraints.AlphaNumKeyConstraint;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
import org.apache.accumulo.test.functional.BadIterator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java b/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
index 4d374eb..18c275a 100644
--- a/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
@@ -75,7 +75,7 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.NamespacePermission;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.test.constraints.NumericValueConstraint;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index c327c28..eae5ca9 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -1692,7 +1692,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
ts.exec("tables", true, "thing2.thingy", false);
// put constraints on a namespace
- ts.exec("constraint -ns thing3 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ ts.exec("constraint -ns thing3 -a org.apache.accumulo.test.constraints.NumericValueConstraint", true);
ts.exec("createtable thing3.constrained", true);
ts.exec("table thing3.constrained", true);
ts.exec("constraint -d 1");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/constraints/AlphaNumKeyConstraint.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/constraints/AlphaNumKeyConstraint.java b/test/src/main/java/org/apache/accumulo/test/constraints/AlphaNumKeyConstraint.java
new file mode 100644
index 0000000..14fa82d
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/constraints/AlphaNumKeyConstraint.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.constraints;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+
+/**
+ * This class is an accumulo constraint that ensures all fields of a key are alpha numeric.
+ */
+public class AlphaNumKeyConstraint implements Constraint {
+
+ static final short NON_ALPHA_NUM_ROW = 1;
+ static final short NON_ALPHA_NUM_COLF = 2;
+ static final short NON_ALPHA_NUM_COLQ = 3;
+
+ static final String ROW_VIOLATION_MESSAGE = "Row was not alpha numeric";
+ static final String COLF_VIOLATION_MESSAGE = "Column family was not alpha numeric";
+ static final String COLQ_VIOLATION_MESSAGE = "Column qualifier was not alpha numeric";
+
+ private boolean isAlphaNum(byte bytes[]) {
+ for (byte b : bytes) {
+ boolean ok = ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9'));
+ if (!ok)
+ return false;
+ }
+
+ return true;
+ }
+
+ private Set<Short> addViolation(Set<Short> violations, short violation) {
+ if (violations == null) {
+ violations = new LinkedHashSet<>();
+ violations.add(violation);
+ } else if (!violations.contains(violation)) {
+ violations.add(violation);
+ }
+ return violations;
+ }
+
+ @Override
+ public List<Short> check(Environment env, Mutation mutation) {
+ Set<Short> violations = null;
+
+ if (!isAlphaNum(mutation.getRow()))
+ violations = addViolation(violations, NON_ALPHA_NUM_ROW);
+
+ Collection<ColumnUpdate> updates = mutation.getUpdates();
+ for (ColumnUpdate columnUpdate : updates) {
+ if (!isAlphaNum(columnUpdate.getColumnFamily()))
+ violations = addViolation(violations, NON_ALPHA_NUM_COLF);
+
+ if (!isAlphaNum(columnUpdate.getColumnQualifier()))
+ violations = addViolation(violations, NON_ALPHA_NUM_COLQ);
+ }
+
+ return null == violations ? null : new ArrayList<>(violations);
+ }
+
+ @Override
+ public String getViolationDescription(short violationCode) {
+
+ switch (violationCode) {
+ case NON_ALPHA_NUM_ROW:
+ return ROW_VIOLATION_MESSAGE;
+ case NON_ALPHA_NUM_COLF:
+ return COLF_VIOLATION_MESSAGE;
+ case NON_ALPHA_NUM_COLQ:
+ return COLQ_VIOLATION_MESSAGE;
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/constraints/MaxMutationSize.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/constraints/MaxMutationSize.java b/test/src/main/java/org/apache/accumulo/test/constraints/MaxMutationSize.java
new file mode 100644
index 0000000..1b68e96
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/constraints/MaxMutationSize.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.constraints;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.Mutation;
+
+/**
+ * Ensure that mutations are a reasonable size: we must be able to fit several in memory at a time.
+ *
+ */
+public class MaxMutationSize implements Constraint {
+ static final long MAX_SIZE = Runtime.getRuntime().maxMemory() >> 8;
+ static final List<Short> empty = Collections.emptyList();
+ static final List<Short> violations = Collections.singletonList(Short.valueOf((short) 0));
+
+ @Override
+ public String getViolationDescription(short violationCode) {
+ return String.format("mutation exceeded maximum size of %d", MAX_SIZE);
+ }
+
+ @Override
+ public List<Short> check(Environment env, Mutation mutation) {
+ if (mutation.estimatedMemoryUsed() < MAX_SIZE)
+ return empty;
+ return violations;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/constraints/NumericValueConstraint.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/constraints/NumericValueConstraint.java b/test/src/main/java/org/apache/accumulo/test/constraints/NumericValueConstraint.java
new file mode 100644
index 0000000..0d1ae86
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/constraints/NumericValueConstraint.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.constraints;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.accumulo.core.constraints.Constraint;
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+
+/**
+ * This class is an accumulo constraint that ensures values are numeric strings.
+ */
+public class NumericValueConstraint implements Constraint {
+
+ static final short NON_NUMERIC_VALUE = 1;
+ static final String VIOLATION_MESSAGE = "Value is not numeric";
+
+ private static final List<Short> VIOLATION_LIST = Collections.unmodifiableList(Arrays.asList(NON_NUMERIC_VALUE));
+
+ private boolean isNumeric(byte bytes[]) {
+ for (byte b : bytes) {
+ boolean ok = (b >= '0' && b <= '9');
+ if (!ok)
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public List<Short> check(Environment env, Mutation mutation) {
+ Collection<ColumnUpdate> updates = mutation.getUpdates();
+
+ for (ColumnUpdate columnUpdate : updates) {
+ if (!isNumeric(columnUpdate.getValue()))
+ return VIOLATION_LIST;
+ }
+
+ return null;
+ }
+
+ @Override
+ public String getViolationDescription(short violationCode) {
+
+ switch (violationCode) {
+ case NON_NUMERIC_VALUE:
+ return "Value is not numeric";
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/examples/simple/dirlist/CountIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/examples/simple/dirlist/CountIT.java b/test/src/main/java/org/apache/accumulo/test/examples/simple/dirlist/CountIT.java
deleted file mode 100644
index 93708a6..0000000
--- a/test/src/main/java/org/apache/accumulo/test/examples/simple/dirlist/CountIT.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.examples.simple.dirlist;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import java.util.ArrayList;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.cli.BatchWriterOpts;
-import org.apache.accumulo.core.cli.ScannerOpts;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.examples.simple.dirlist.FileCount;
-import org.apache.accumulo.examples.simple.dirlist.FileCount.Opts;
-import org.apache.accumulo.examples.simple.dirlist.Ingest;
-import org.apache.accumulo.examples.simple.dirlist.QueryUtil;
-import org.apache.accumulo.test.functional.ConfigurableMacBase;
-import org.apache.hadoop.io.Text;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CountIT extends ConfigurableMacBase {
-
- private Connector conn;
- private String tableName;
-
- @Before
- public void setupInstance() throws Exception {
- tableName = getUniqueNames(1)[0];
- conn = getConnector();
- conn.tableOperations().create(tableName);
- BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
- ColumnVisibility cv = new ColumnVisibility();
- // / has 1 dir
- // /local has 2 dirs 1 file
- // /local/user1 has 2 files
- bw.addMutation(Ingest.buildMutation(cv, "/local", true, false, true, 272, 12345, null));
- bw.addMutation(Ingest.buildMutation(cv, "/local/user1", true, false, true, 272, 12345, null));
- bw.addMutation(Ingest.buildMutation(cv, "/local/user2", true, false, true, 272, 12345, null));
- bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 12345, null));
- bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 23456, null));
- bw.addMutation(Ingest.buildMutation(cv, "/local/user1/file1", false, false, false, 2024, 12345, null));
- bw.addMutation(Ingest.buildMutation(cv, "/local/user1/file2", false, false, false, 1028, 23456, null));
- bw.close();
- }
-
- @Test
- public void test() throws Exception {
- Scanner scanner = conn.createScanner(tableName, new Authorizations());
- scanner.fetchColumn(new Text("dir"), new Text("counts"));
- assertFalse(scanner.iterator().hasNext());
-
- Opts opts = new Opts();
- ScannerOpts scanOpts = new ScannerOpts();
- BatchWriterOpts bwOpts = new BatchWriterOpts();
- opts.instance = conn.getInstance().getInstanceName();
- opts.zookeepers = conn.getInstance().getZooKeepers();
- opts.setTableName(tableName);
- opts.setPrincipal(conn.whoami());
- opts.setPassword(new Opts.Password(ROOT_PASSWORD));
- FileCount fc = new FileCount(opts, scanOpts, bwOpts);
- fc.run();
-
- ArrayList<Pair<String,String>> expected = new ArrayList<>();
- expected.add(new Pair<>(QueryUtil.getRow("").toString(), "1,0,3,3"));
- expected.add(new Pair<>(QueryUtil.getRow("/local").toString(), "2,1,2,3"));
- expected.add(new Pair<>(QueryUtil.getRow("/local/user1").toString(), "0,2,0,2"));
- expected.add(new Pair<>(QueryUtil.getRow("/local/user2").toString(), "0,0,0,0"));
-
- int i = 0;
- for (Entry<Key,Value> e : scanner) {
- assertEquals(e.getKey().getRow().toString(), expected.get(i).getFirst());
- assertEquals(e.getValue().toString(), expected.get(i).getSecond());
- i++;
- }
- assertEquals(i, expected.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e0f19a1/test/src/main/java/org/apache/accumulo/test/examples/simple/filedata/ChunkInputFormatIT.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/examples/simple/filedata/ChunkInputFormatIT.java b/test/src/main/java/org/apache/accumulo/test/examples/simple/filedata/ChunkInputFormatIT.java
deleted file mode 100644
index cb53ec0..0000000
--- a/test/src/main/java/org/apache/accumulo/test/examples/simple/filedata/ChunkInputFormatIT.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.accumulo.test.examples.simple.filedata;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.examples.simple.filedata.ChunkInputFormat;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-
-public class ChunkInputFormatIT extends AccumuloClusterHarness {
-
- // track errors in the map reduce job; jobs insert a dummy error for the map and cleanup tasks (to ensure test correctness),
- // so error tests should check to see if there is at least one error (could be more depending on the test) rather than zero
- private static Multimap<String,AssertionError> assertionErrors = ArrayListMultimap.create();
-
- private static final Authorizations AUTHS = new Authorizations("A", "B", "C", "D");
-
- private static List<Entry<Key,Value>> data;
- private static List<Entry<Key,Value>> baddata;
-
- private Connector conn;
- private String tableName;
-
- @Before
- public void setupInstance() throws Exception {
- conn = getConnector();
- tableName = getUniqueNames(1)[0];
- conn.securityOperations().changeUserAuthorizations(conn.whoami(), AUTHS);
- }
-
- @BeforeClass
- public static void setupClass() {
- System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
-
- data = new ArrayList<>();
- ChunkInputStreamIT.addData(data, "a", "refs", "ida\0ext", "A&B", "ext");
- ChunkInputStreamIT.addData(data, "a", "refs", "ida\0name", "A&B", "name");
- ChunkInputStreamIT.addData(data, "a", "~chunk", 100, 0, "A&B", "asdfjkl;");
- ChunkInputStreamIT.addData(data, "a", "~chunk", 100, 1, "A&B", "");
- ChunkInputStreamIT.addData(data, "b", "refs", "ida\0ext", "A&B", "ext");
- ChunkInputStreamIT.addData(data, "b", "refs", "ida\0name", "A&B", "name");
- ChunkInputStreamIT.addData(data, "b", "~chunk", 100, 0, "A&B", "qwertyuiop");
- ChunkInputStreamIT.addData(data, "b", "~chunk", 100, 0, "B&C", "qwertyuiop");
- ChunkInputStreamIT.addData(data, "b", "~chunk", 100, 1, "A&B", "");
- ChunkInputStreamIT.addData(data, "b", "~chunk", 100, 1, "B&C", "");
- ChunkInputStreamIT.addData(data, "b", "~chunk", 100, 1, "D", "");
- baddata = new ArrayList<>();
- ChunkInputStreamIT.addData(baddata, "c", "refs", "ida\0ext", "A&B", "ext");
- ChunkInputStreamIT.addData(baddata, "c", "refs", "ida\0name", "A&B", "name");
- }
-
- public static void entryEquals(Entry<Key,Value> e1, Entry<Key,Value> e2) {
- assertEquals(e1.getKey(), e2.getKey());
- assertEquals(e1.getValue(), e2.getValue());
- }
-
- public static class CIFTester extends Configured implements Tool {
- public static class TestMapper extends Mapper<List<Entry<Key,Value>>,InputStream,List<Entry<Key,Value>>,InputStream> {
- int count = 0;
-
- @Override
- protected void map(List<Entry<Key,Value>> key, InputStream value, Context context) throws IOException, InterruptedException {
- String table = context.getConfiguration().get("MRTester_tableName");
- assertNotNull(table);
-
- byte[] b = new byte[20];
- int read;
- try {
- switch (count) {
- case 0:
- assertEquals(key.size(), 2);
- entryEquals(key.get(0), data.get(0));
- entryEquals(key.get(1), data.get(1));
- assertEquals(read = value.read(b), 8);
- assertEquals(new String(b, 0, read), "asdfjkl;");
- assertEquals(read = value.read(b), -1);
- break;
- case 1:
- assertEquals(key.size(), 2);
- entryEquals(key.get(0), data.get(4));
- entryEquals(key.get(1), data.get(5));
- assertEquals(read = value.read(b), 10);
- assertEquals(new String(b, 0, read), "qwertyuiop");
- assertEquals(read = value.read(b), -1);
- break;
- default:
- fail();
- }
- } catch (AssertionError e) {
- assertionErrors.put(table, e);
- } finally {
- value.close();
- }
- count++;
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- String table = context.getConfiguration().get("MRTester_tableName");
- assertNotNull(table);
-
- try {
- assertEquals(2, count);
- } catch (AssertionError e) {
- assertionErrors.put(table, e);
- }
- }
- }
-
- public static class TestNoClose extends Mapper<List<Entry<Key,Value>>,InputStream,List<Entry<Key,Value>>,InputStream> {
- int count = 0;
-
- @Override
- protected void map(List<Entry<Key,Value>> key, InputStream value, Context context) throws IOException, InterruptedException {
- String table = context.getConfiguration().get("MRTester_tableName");
- assertNotNull(table);
-
- byte[] b = new byte[5];
- int read;
- try {
- switch (count) {
- case 0:
- assertEquals(read = value.read(b), 5);
- assertEquals(new String(b, 0, read), "asdfj");
- break;
- default:
- fail();
- }
- } catch (AssertionError e) {
- assertionErrors.put(table, e);
- }
- count++;
- try {
- context.nextKeyValue();
- fail();
- } catch (IOException ioe) {
- assertionErrors.put(table + "_map_ioexception", new AssertionError(toString(), ioe));
- }
- }
- }
-
- public static class TestBadData extends Mapper<List<Entry<Key,Value>>,InputStream,List<Entry<Key,Value>>,InputStream> {
- @Override
- protected void map(List<Entry<Key,Value>> key, InputStream value, Context context) throws IOException, InterruptedException {
- String table = context.getConfiguration().get("MRTester_tableName");
- assertNotNull(table);
-
- byte[] b = new byte[20];
- try {
- assertEquals(key.size(), 2);
- entryEquals(key.get(0), baddata.get(0));
- entryEquals(key.get(1), baddata.get(1));
- } catch (AssertionError e) {
- assertionErrors.put(table, e);
- }
- try {
- assertFalse(value.read(b) > 0);
- try {
- fail();
- } catch (AssertionError e) {
- assertionErrors.put(table, e);
- }
- } catch (Exception e) {
- // expected, ignore
- }
- try {
- value.close();
- try {
- fail();
- } catch (AssertionError e) {
- assertionErrors.put(table, e);
- }
- } catch (Exception e) {
- // expected, ignore
- }
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- if (args.length != 2) {
- throw new IllegalArgumentException("Usage : " + CIFTester.class.getName() + " <table> <mapperClass>");
- }
-
- String table = args[0];
- assertionErrors.put(table, new AssertionError("Dummy"));
- assertionErrors.put(table + "_map_ioexception", new AssertionError("Dummy_ioexception"));
- getConf().set("MRTester_tableName", table);
-
- Job job = Job.getInstance(getConf());
- job.setJobName(this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
- job.setJarByClass(this.getClass());
-
- job.setInputFormatClass(ChunkInputFormat.class);
-
- ChunkInputFormat.setZooKeeperInstance(job, getCluster().getClientConfig());
- ChunkInputFormat.setConnectorInfo(job, getAdminPrincipal(), getAdminToken());
- ChunkInputFormat.setInputTableName(job, table);
- ChunkInputFormat.setScanAuthorizations(job, AUTHS);
-
- @SuppressWarnings("unchecked")
- Class<? extends Mapper<?,?,?,?>> forName = (Class<? extends Mapper<?,?,?,?>>) Class.forName(args[1]);
- job.setMapperClass(forName);
- job.setMapOutputKeyClass(Key.class);
- job.setMapOutputValueClass(Value.class);
- job.setOutputFormatClass(NullOutputFormat.class);
-
- job.setNumReduceTasks(0);
-
- job.waitForCompletion(true);
-
- return job.isSuccessful() ? 0 : 1;
- }
-
- public static int main(String... args) throws Exception {
- Configuration conf = new Configuration();
- conf.set("mapreduce.framework.name", "local");
- conf.set("mapreduce.cluster.local.dir", new File(System.getProperty("user.dir"), "target/mapreduce-tmp").getAbsolutePath());
- return ToolRunner.run(conf, new CIFTester(), args);
- }
- }
-
- @Test
- public void test() throws Exception {
- conn.tableOperations().create(tableName);
- BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
-
- for (Entry<Key,Value> e : data) {
- Key k = e.getKey();
- Mutation m = new Mutation(k.getRow());
- m.put(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
- bw.addMutation(m);
- }
- bw.close();
-
- assertEquals(0, CIFTester.main(tableName, CIFTester.TestMapper.class.getName()));
- assertEquals(1, assertionErrors.get(tableName).size());
- }
-
- @Test
- public void testErrorOnNextWithoutClose() throws Exception {
- conn.tableOperations().create(tableName);
- BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
-
- for (Entry<Key,Value> e : data) {
- Key k = e.getKey();
- Mutation m = new Mutation(k.getRow());
- m.put(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
- bw.addMutation(m);
- }
- bw.close();
-
- assertEquals(1, CIFTester.main(tableName, CIFTester.TestNoClose.class.getName()));
- assertEquals(1, assertionErrors.get(tableName).size());
- // this should actually exist, in addition to the dummy entry
- assertEquals(2, assertionErrors.get(tableName + "_map_ioexception").size());
- }
-
- @Test
- public void testInfoWithoutChunks() throws Exception {
- conn.tableOperations().create(tableName);
- BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
- for (Entry<Key,Value> e : baddata) {
- Key k = e.getKey();
- Mutation m = new Mutation(k.getRow());
- m.put(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
- bw.addMutation(m);
- }
- bw.close();
-
- assertEquals(0, CIFTester.main(tableName, CIFTester.TestBadData.class.getName()));
- assertEquals(1, assertionErrors.get(tableName).size());
- }
-}